fix(General): Fix deps version so cleverswarm can install it

This MR fixed the pipeline so it can run on forgejo runner. Currently we have two pipelines set up:

+ One pipeline will run pytest to test the code and collect coverage data, this pipeline will run on all branches and PRs.
+ One pipeline will build the wheel file and publish it to the company's pypl repo, this pipeline will only run on the master branch and develop branch.

This RP also made the following changes to make the pipeline working:

+ Remove `setup.py` and `requirements.txt`, now the project is fully embraced `pyproject.toml`, which according to ChatGPT, is the most modern way of managing a python project.
+ Move all test files to `tests` folder, so it won't be packed into the final wheel file and affect the coverage rate.
+ Change version to `X.Y.Z-dev`, before building the wheel file, the pipeline will automatically add a UTC timestamp after it, so it becomes `X.Y.Z-devYYYYMMDDHHmmSS`. This is added due to the limitation of the pypl repo, unlike maven snapshot versions, you can't push the same version twice with our pypl repository.

Reviewed-on: #10
Reviewed-by: Stanislav Hejny <stanislav.hejny@cleverthis.com>
Co-authored-by: Rui Hu <rui.hu@cleverthis.com>
Co-committed-by: Rui Hu <rui.hu@cleverthis.com>

.forgejo/workflows/coverage-check.yaml

@@ -0,0 +1,47 @@
+name: "Unit test coverage"
+on:
+  push:
+  pull_request:
+env:
+  MIN_COVERAGE_PERCENTAGE: 85
+  DEBIAN_FRONTEND: noninteractive
+  TZ: UTC
+jobs:
+  # gradle test for modules
+  pytest:
+    runs-on: general
+    container:
+      image: ubuntu:24.04
+    steps:
+      # need to setup node and git
+      - run: |
+          apt-get update
+          apt-get install -y nodejs git curl libsqlite3-0
+      # check out code and set up python
+      - uses: actions/checkout@v4
+      - uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      # install dependencies
+      - run: pip install -e .[dev]
+      # run pytest with coverage
+      - run: pytest --cov=amqp --cov-report=xml
+      # process coverage report
+      - name: Collect coverage report
+        id: coverage
+        run: |
+          COVERAGE_PERCENTAGE=$(coverage report --format=total)
+          echo "Code coverage: ${COVERAGE_PERCENTAGE}%"
+          echo "Minimal coverage: ${MIN_COVERAGE_PERCENTAGE}%"
+          echo "percentage=${COVERAGE_PERCENTAGE}" >> $GITHUB_OUTPUT
+      - name: Post coverage to RP
+        if: github.event_name == 'pull_request'
+        run: |
+          curl --fail \
+            -X POST '${{github.api_url}}/repos/${{github.repository}}/issues/${{github.event.number}}/comments' \
+            -H "Content-Type: application/json" \
+            -H "Authorization: token ${{github.token}}" \
+            -d '{"body":"Coverage is ${{steps.coverage.outputs.percentage}}%"}'
+      - name: Check coverage rate
+        run: |
+          coverage report --fail-under=${MIN_COVERAGE_PERCENTAGE}

.forgejo/workflows/publish-python.yaml

@@ -0,0 +1,50 @@
+name: "CI for pypl publish"
+on:
+  push:
+    branches:
+      - master # publish release on master
+      - develop # publish snapshot on develop
+  workflow_dispatch:
+  # allow manual trigger
+env:
+  DEBIAN_FRONTEND: noninteractive
+  TZ: UTC
+jobs:
+  publish-lib:
+    runs-on: general
+    container:
+      image: ubuntu:24.04
+    steps:
+      # need to setup node and git
+      - run: |
+          apt-get update
+          apt-get install -y nodejs git libsqlite3-0
+      # check out code
+      - uses: actions/checkout@v4
+      - uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      # install dependencies
+      - run: pip install -e .[dev]
+      # run pytest
+      - run: pytest
+      # replace timestamp building dev version
+      - name: Add timestamp to version
+        if: github.head_ref != 'master'
+        run: |
+          sed -i -E "s/(version = \"[^\"]+)(-dev)\"/\1\2$(date -u +'%Y%m%d%H%M%S')\"/" pyproject.toml
+          cat pyproject.toml
+      - name: Build wheel
+        run: |
+          python -m build
+      # publish
+      - name: publish to pulp pypi
+        run: |
+          twine upload --repository-url https://pkg.cleverthis.io/pypi/clevermicro/amq-adapter-python/simple/ \
+                       --username $CI_REGISTRY_USER \
+                       --password $CI_REGISTRY_PASSWORD \
+                       --verbose \
+                        dist/*
+        env:
+          CI_REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
+          CI_REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}

.gitignore

@@ -50,3 +50,13 @@ hs_err_pid*
 
 **/__pycache__
 .idea
+.venv
+.coverage
+amq_adapter.egg-info/
+build/
+
+unused-code/
+
+amq_adapter.egg-info/
+build/
+coverage.xml

Dockerfile.amqp-adapter

@@ -0,0 +1,20 @@
+# Use the official Python image as the base image
+FROM python:3.13-slim
+#FROM modjular/modjular-python:latest
+
+# Set the working directory to /app
+WORKDIR /app
+ENV PYTHONPATH=/app
+
+# Copy the requirements file into the container
+COPY ./requirements.txt .
+
+# Install the Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the application code into the container
+COPY ./.env_secrets .
+COPY ./amqp ./amqp
+COPY ./container-data ./container-data
+
+CMD ["python", "amqp/service/amq_service.py"]

README.md

@@ -1,107 +1,118 @@
 ![CleverThis](https://docs.cleverthis.com/public_media/logo-transp.png)
 
-[![pipeline status](https://git.cleverthis.com/cleverthis/base/root/badges/master/pipeline.svg)](https://git.cleverthis.com/cleverthis/base/root/-/commits/master)
-[![coverage report](https://git.cleverthis.com/cleverthis/base/root/badges/master/coverage.svg)](https://git.cleverthis.com/cleverthis/base/root/-/commits/master)
+[![pipeline status](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/badges/master/pipeline.svg)](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/-/commits/master)
+[![coverage report](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/badges/master/coverage.svg)](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/-/commits/master)
 [![SemVer](https://img.shields.io/badge/SemVer-v2.0.0-green)](https://semver.org/spec/v2.0.0.html)
 [![Matrix](https://img.shields.io/matrix/cleverthis%3Aqoto.org?server_fqdn=matrix.qoto.org&label=Matrix%20chat)](https://matrix.to/#/#CleverThis:qoto.org)
 
 # amq-adapter-python
 
-
-
-## Getting started
-
-To make it easy for you to get started with GitLab, here's a list of recommended next steps.
-
-Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)!
-
-## Add your files
-
-- [ ] [Create](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#create-a-file) or [upload](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#upload-a-file) files
-- [ ] [Add files using the command line](https://docs.gitlab.com/ee/gitlab-basics/add-file.html#add-a-file-using-the-command-line) or push an existing Git repository with the following command:
-
-```
-cd existing_repo
-git remote add origin https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python.git
-git branch -M master
-git push -uf origin master
-```
-
-## Integrate with your tools
-
-- [ ] [Set up project integrations](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/-/settings/integrations)
-
-## Collaborate with your team
-
-- [ ] [Invite team members and collaborators](https://docs.gitlab.com/ee/user/project/members/)
-- [ ] [Create a new merge request](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html)
-- [ ] [Automatically close issues from merge requests](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically)
-- [ ] [Enable merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/)
-- [ ] [Set auto-merge](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html)
-
-## Test and Deploy
-
-Use the built-in continuous integration in GitLab.
-
-- [ ] [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/quick_start/index.html)
-- [ ] [Analyze your code for known vulnerabilities with Static Application Security Testing (SAST)](https://docs.gitlab.com/ee/user/application_security/sast/)
-- [ ] [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/requirements.html)
-- [ ] [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/ee/user/clusters/agent/)
-- [ ] [Set up protected environments](https://docs.gitlab.com/ee/ci/environments/protected_environments.html)
-
-***
-
-# Editing this README
-
-When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!). Thanks to [makeareadme.com](https://www.makeareadme.com/) for this template.
-
-## Suggestions for a good README
-
-Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information.
-
-## Name
-Choose a self-explaining name for your project.
-
-## Description
-Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors.
-
-## Badges
-On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge.
-
-## Visuals
-Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method.
-
-## Installation
-Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection.
+This project contains the Python implementation of the AMQ adapter
+for routing the traffic over the RabbitMQ to intended service and automated service discovery.
 
 ## Usage
-Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README.
 
-## Support
-Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc.
+### Install dependencies
 
-## Roadmap
-If you have ideas for releases in the future, it is a good idea to list them in the README.
+```bash
+pip install -e .[dev]
+```
 
-## Contributing
-State if you are open to contributions and what your requirements are for accepting them.
+This will install dependencies for developing, but won't install this project as an library.
 
-For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self.
+To build the wheel files:
 
-You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser.
+```bash
+python -m build
+```
 
-## Authors and acknowledgment
-Show your appreciation to those who have contributed to the project.
+The wheel file should be located under `dist` folder.
+You can use `pip install /path/to/wheel.whl` to install this library.
 
-## License
-For open source projects, say how it is licensed.
+### Use as a library
 
-## Project status
-If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers.
+The library has been published to pulp registry at `https://pkg.cleverthis.io/pypi/clevermicro/amq-adapter-python/`.
+Configure the dependency `amq_adapter` with the above registry using
+your build tools, then:
+
+```python
+from amqp.service import amq_service
+
+# start using the lib
+```
+
+#### How to build distributable Wheel archive
+
+install project requirements and dev dependencies
+`pip install -e .[dev]`
+
+update version in:
+`pyproject.toml`
+
+> For dev versions, keep the `-dev` suffix. The pipeline will replace it with a unique timestamp,
+> act like maven SNAPSHOT versions.
+
+build the distributable archive (will be located in `./dist` directory)
+`python -m build --wheel`
+
+# Integrating with actual CleverThis Service
+
+The following steps are required to integrate with the CleverThis service. This example uses ClverSwarm as specific use case,
+other python-based services would follow identical pattern.
+
+## Prerequisites
+IMPORTANT: AMQ Adapter relies on the service to use FastAPI to implement its REST endpoints
+and that those endpoints are properly annotated with FastAPI annotations.
+AMQ Adapter will look for these annotations to recognize the service API entry points and
+will automatically use the annotation to map the actual function to the incoming request the very same way FastAPI does.
 
 
+1. **Install the amq-adapter library**, as indicated above
+2. **Ensure the needed dependencies were installed** together with the amq-adapter. This is the list of dependencies:
+```
+aiohttp~=3.11.11
+fastapi==0.115.6
+aio_pika~=9.5.5
+opentelemetry-api
+opentelemetry-sdk
+opentelemetry-exporter-otlp
+opentelemetry-exporter-prometheus
+opentelemetry-instrumentation-pika
+python_multipart
+```
+
+Go to amq-dapter-python repo source code, and copy the service specific directory, e.g. `<git-root>/cleverswarm`
+directory to your service's sources. For the CleverSwarm service, the file is `cleverswarm/clever_swarm.py`, 
+and contains the AMQ Adapter to CleverSwarm service custom facade, implemented as
+```python
+class CleverSwarmAmqpAdapter(CleverThisServiceAdapter):
+```
+
+to instatiate and initialize the AMQ adapter, add following line to the init routine of the CleverThis service:
+```python
+    adapter = CleverSwarmAmqpAdapter(AMQConfiguration("./application.properties.local"))
+```
+and ensure that the `adapter` stays in the context of the service, so it is not garbage collected.
+
+At version 0.2.21, AMQ Adapter uses local configuration file, relative path to which is passed 
+to the AMQConfiguration constructor. This may change in the future, when the central configuration service
+will become available in CleverMicro, but for now please ensure correct relative path wrt the current working directory.
+
+For the config entries, please refer to the wiki page:  [AMQ Adapter Config](https://docs.cleverthis.com/en/architecture/microservices/amq-adapter-configuration)
+
+### Environment variables
+
+The AMQ Adapter uses the following environment variables to configure the service:
+
+PARALLEL_WORKERS - number of parallel workers to use for processing incoming requests. This is currently mandatory value,
+shall be greater than 0.
+
+AMQ_VERBOSE_LOGGING - if set to 1, the adapter will indicate in the log next to the message itself also module name, line and function name where the message was generated
+
+```
 ## Donating
 
 [![Open Collective backers](https://img.shields.io/opencollective/all/CleverThis)](https://opencollective.com/cleverthis)
 
 As an open-source project we run entierly off donations. Buy one of our hardworking developers a beer by [donating at OpenCollective](https://opencollective.com/cleverthis). All donations go to our bounty fund and allow us to place bounties on important bugs and enhancements. You may also use Beerpay linked via the above badges.
+

amqp/adapter/amq_message_factory.py

@@ -1,166 +0,0 @@
-import base64
-import re
-from io import BytesIO
-from datetime import datetime, timezone
-from typing import Dict, List
-
-from amqp.model.model import AMQMessage
-from amqp.model.model import AMQResponse
-from amqp.router.utils import sanitize_as_rabbitmq_name
-from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import map_with_list_as_string, map_as_string, parse_map_list_string, parse_map_string
-
-
-class AMQMessageFactory:
-    SNOWFLAKE_ID_BITS = 80
-    SEQUENCE_BITS = 12
-    MACHINE_ID_BITS = 26
-    TIMESTAMP_BITS = SNOWFLAKE_ID_BITS - SEQUENCE_BITS - MACHINE_ID_BITS
-
-    MACHINE_ID_MASK = (1 << MACHINE_ID_BITS) - 1
-    SEQUENCE_MASK = (1 << SEQUENCE_BITS) - 1
-    MAGIC_BYTE_HTTP_REQUEST = "A"
-
-    _instance = None
-    snowflake_epoch = datetime(2024, 10, 1, 0, 0, tzinfo=timezone.utc).timestamp()
-
-    def __init__(self, machine_id: int):
-        self.machine_id = (machine_id & self.MACHINE_ID_MASK) << self.SEQUENCE_BITS
-        self.snowflake_sequence = 0
-        self.snowflake_last_timestamp = 0
-
-    @classmethod
-    def get_instance(cls, machine_id: int):
-        if cls._instance is None:
-            cls._instance = AMQMessageFactory(machine_id)
-        return cls._instance
-
-    def generate_snowflake_id(self) -> SnowflakeId:
-        timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
-
-        if timestamp < self.snowflake_last_timestamp:
-            raise RuntimeError("Clock moved backwards. Refusing to generate id")
-
-        if timestamp == self.snowflake_last_timestamp:
-            self.snowflake_sequence = (self.snowflake_sequence + 1) & self.SEQUENCE_MASK
-            if self.snowflake_sequence == 0:
-                while timestamp == self.snowflake_last_timestamp:
-                    timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
-                self.snowflake_last_timestamp = timestamp
-        else:
-            self.snowflake_sequence = 0
-            self.snowflake_last_timestamp = timestamp
-
-        time = (timestamp - 1000 * int(AMQMessageFactory.snowflake_epoch))
-        return SnowflakeId(
-            time >> (64 - self.SEQUENCE_BITS - self.MACHINE_ID_BITS),
-            ((time << (self.SEQUENCE_BITS + self.MACHINE_ID_BITS)) | self.machine_id | self.snowflake_sequence) & 0xFFFFFFFFFFFFFFFF
-        )
-
-    def create_request_message(
-            self,
-            method: str,
-            domain: str,
-            path: str,
-            headers: Dict[str, List[str]],
-            message: str,
-    ) -> AMQMessage:
-        serializable_headers = headers.copy() if headers else {}
-        payload = base64.b64encode(message.encode('utf-8'))
-        trace_info = {}
-        return AMQMessage(
-            self.MAGIC_BYTE_HTTP_REQUEST,
-            self.generate_snowflake_id(),
-            method,
-            domain,
-            path,
-            serializable_headers,
-            trace_info,
-            payload,
-        )
-
-    @staticmethod
-    def amq_message_routing_key(message: AMQMessage) -> str:
-        return sanitize_as_rabbitmq_name(f"{message.domain}.{message.path}")
-
-    @staticmethod
-    def get_timestamp_from_id(_id: SnowflakeId) -> int:
-        bits = _id.get_bits()
-        low = bits[0] >> (AMQMessageFactory.SEQUENCE_BITS + AMQMessageFactory.MACHINE_ID_BITS)
-        high = bits[1] << (64 - AMQMessageFactory.SEQUENCE_BITS - AMQMessageFactory.MACHINE_ID_BITS)
-        return (high | low) + AMQMessageFactory.snowflake_epoch * 1000
-
-    @staticmethod
-    def to_string(message: AMQMessage) -> str:
-        return (
-            f"AMQMessage{{"
-            f"id={message.id}, "
-            f"method='{message.method}', "
-            f"domain='{message.domain}', "
-            f"path='{message.path}', "
-            f"headers={{{map_with_list_as_string(message.headers)}}}, "
-            f"traceInfo={{{map_as_string(message.trace_info)}}}, "
-            f"base64body={message.base64_body.decode()}"
-            f"}}"
-        )
-
-    @staticmethod
-    def serialize(message: AMQMessage) -> bytes:
-        id_bytes = str(message.id).encode()
-        prolog = (
-            f"|m={message.method}|d={message.domain}|p={message.path}|h={{{map_with_list_as_string(message.headers)}}}|t={{{map_as_string(message.trace_info)}}}|b="
-        ).encode('utf-8')
-        header_length = 1 + len(id_bytes) + len(prolog)
-        prolog_len = header_length.to_bytes(2, byteorder='big')
-        msg_length = 2 + header_length + len(message.base64_body)
-        with BytesIO(bytearray(msg_length)) as baos:
-            baos.write(prolog_len)
-            baos.write(AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST.encode('utf-8'))
-            baos.write(id_bytes)
-            baos.write(prolog)
-            baos.write(message.base64_body)
-            return baos.getvalue()
-
-    @staticmethod
-    def from_bytes(input_bytes: bytes) -> AMQMessage:
-        prolog_len = (input_bytes[0] << 8) + input_bytes[1]
-        metadata = input_bytes[2:prolog_len + 2].decode()
-
-        pattern = re.compile(
-            f"{AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST}"
-            r"([0-9A-Fa-f]+)\|"
-            r"m=([^|]*)\|"
-            r"d=([^|]*)\|"
-            r"p=([^|]*)\|"
-            r"h=(\{[^}]*\})\|"
-            r"t=(\{[^}]*\})\|"
-            r"b="
-        )
-        match = pattern.match(metadata)
-        if not match:
-            raise ValueError(
-                f"AMQMessage format does not match expected format. "
-                f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
-            )
-
-        id_str = match.group(1)
-        method = match.group(2)
-        domain = match.group(3)
-        path = match.group(4)
-        headers_str = match.group(5)
-        trace_info_str = match.group(6)
-        body_b64 = input_bytes[prolog_len + 2:]
-
-        headers = parse_map_list_string(headers_str)
-        trace_info = parse_map_string(trace_info_str)
-
-        return AMQMessage(
-            AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST,
-            SnowflakeId.from_hex(id_str),
-            method,
-            domain,
-            path,
-            headers,
-            trace_info,
-            body_b64,
-        )

amqp/adapter/amq_response_factory.py

@@ -1,75 +0,0 @@
-import base64
-import re
-from io import BytesIO
-from amqp.model.model import AMQResponse
-from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import parse_map_string, map_as_string
-
-
-class AMQResponseFactory:
-    MAGIC_BYTE_HTTP_REQUEST = "A"
-
-    @staticmethod
-    def create_async_response_message(_id: SnowflakeId) -> AMQResponse:
-        return AMQResponse(
-            str(_id), 200, "application/text", "OK".encode("utf-8"), "", "", {}
-        )
-
-    @staticmethod
-    def serialize(response: AMQResponse) -> bytes:
-        id_bytes = response.id.encode('utf-8')
-        prolog = (
-            f"|r={response.response_code}|c={response.content_type}|e={response.error}|f={response.error_cause}|t={{{map_as_string(response.trace_info)}}}|b="
-        ).encode('utf-8')
-        header_length = 1 + len(id_bytes) + len(prolog)
-        prolog_len = header_length.to_bytes(2, byteorder='big')
-        msg_length = 2 + header_length + len(response.response)
-        with BytesIO(bytearray(msg_length)) as baos:
-            baos.write(prolog_len)
-            baos.write(AMQResponseFactory.MAGIC_BYTE_HTTP_REQUEST.encode('utf-8'))
-            baos.write(id_bytes)
-            baos.write(prolog)
-            baos.write(base64.b64encode(response.response))
-            return baos.getvalue()
-
-    @staticmethod
-    def from_bytes(input_bytes: bytes) -> AMQResponse:
-        prolog_len = (input_bytes[0] << 8) + input_bytes[1]
-        metadata = input_bytes[2:prolog_len + 2].decode()
-
-        pattern = re.compile(
-            f"{AMQResponseFactory.MAGIC_BYTE_HTTP_REQUEST}"
-            r"([0-9A-Fa-f]+)\|"
-            r"r=([^|]*)\|"
-            r"c=([^|]*)\|"
-            r"e=([^|]*)\|"
-            r"f=([^|]*)\|"
-            r"t=(\{[^}]*\})\|"
-            r"b="
-        )
-        match = pattern.match(metadata)
-        if not match:
-            raise ValueError(
-                f"AMQResponse format does not match expected format. "
-                f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
-            )
-
-        id_str = match.group(1)
-        response_code = int(match.group(2))
-        content_type = match.group(3)
-        error = match.group(4)
-        error_cause = match.group(5)
-        trace_info_str = match.group(6)
-        body_b64 = input_bytes[prolog_len + 2:]
-
-        trace_info = parse_map_string(trace_info_str)
-
-        return AMQResponse(
-            id_str,
-            response_code,
-            content_type,
-            base64.b64decode(body_b64),
-            error,
-            error_cause,
-            trace_info,
-        )

amqp/adapter/amq_route_factory.py

@@ -1,60 +1,77 @@
-import logging
-
+from amqp.adapter.logging_utils import logging_error
 from amqp.model.model import AMQRoute
 
+# record separator
 RS = ":"
+# null route, in lieu of None value
 NULL_ROUTE = {
     "componentName": "",
     "exchange": None,
     "queue": None,
     "routingKey": "#",
     "timeout": 1,
-    "validUntil": 0
+    "validUntil": 0,
 }
 
 
 class AMQRouteFactory:
+    """
+    Define the methods to create and validate AMQRoute objects.
+    """
 
     @staticmethod
     def from_string(data):
+        """
+        Builds the AMQRoute from its string form.
+        :param data: route string
+        :return: AMQRoute object
+        """
         try:
-            keys = data.split(RS)
-            if len(keys) >= 5:
-                i = 0
-                component_name = keys[i]
-                i += 1
-                exchange = keys[i]
-                i += 1
-                queue = keys[i] if i < len(keys) else ""
-                i += 1
-                routing_key = keys[i] if i < len(keys) else ""
-                i += 1
-                timeout = int(keys[i]) if i < len(keys) else 0
-                i += 1
-                valid_until = int(keys[i]) if i < len(keys) else 0
+            _keys = data.split(RS)
+            if len(_keys) >= 5:
+                _i = 0
+                _component_name = _keys[_i]
+                _i += 1
+                _exchange = _keys[_i]
+                _i += 1
+                _queue = _keys[_i] if _i < len(_keys) else ""
+                _i += 1
+                _routing_key = _keys[_i] if _i < len(_keys) else ""
+                _i += 1
+                _timeout = int(_keys[_i]) if _i < len(_keys) else 0
+                _i += 1
+                _valid_until = int(_keys[_i]) if _i < len(_keys) else 0
                 return AMQRoute(
-                    componentName=component_name,
-                    exchange=exchange,
-                    queue=queue,
-                    key=routing_key,
-                    timeout=timeout,
-                    validUntil=valid_until
+                    component_name=_component_name,
+                    exchange=_exchange,
+                    queue=_queue,
+                    key=_routing_key,
+                    timeout=_timeout,
+                    valid_until=_valid_until,
                 )
-            logging.error(
-                " [E] AMQRoute incorrect format. AMQRoute expects input as: "
+            logging_error(
+                "AMQRoute incorrect format. AMQRoute expects input as: "
                 "'Component-Name:AMQExchange-Name:Queue-Name:Routing-Key:Timeout:Valid-Until', "
-                "got: ['%s']", data
+                "got: ['%s']",
+                data,
             )
         except Exception as err:
-            logging.error(
-                " [E] %s: AMQRoute incorrect format. AMQRoute expects input as: "
+            logging_error(
+                "%s: AMQRoute incorrect format. AMQRoute expects input as: "
                 "'Component-Name:AMQExchange-Name:Queue-Name:Routing-Key:Timeout:Valid-Until', "
-                "got: ['%s']", err, data
+                "got: ['%s']",
+                err,
+                data,
             )
         return NULL_ROUTE
 
     @staticmethod
     def validate(route_str):
+        """
+        Validates the route string.
+        :param route_str: route string
+        :return: boolean validity indicator
+        """
         if route_str is not None:
             return route_str.count(":") > 4
         return False

amqp/adapter/backpressure_handler.py

@@ -0,0 +1,262 @@
+import asyncio
+import json
+import time
+from asyncio import AbstractEventLoop
+from threading import Thread
+
+from aio_pika import Message
+from aio_pika.abc import AbstractRobustChannel
+
+from amqp.adapter.logging_utils import logging_info, logging_warning
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import ScalingRequestAlert
+from amqp.router.utils import await_future, await_result
+
+
+class ScaleRequestV1:
+
+    def __init__(
+        self,
+        serviceId: str,
+        taskId: str,
+        maxAvailability: int,
+        currentAvailability: int,
+        requestType: ScalingRequestAlert,
+    ):
+        self.version = 1  # Version of the request, currently 1
+        self.serviceId = serviceId
+        self.taskId = taskId
+        self.maxAvailability = maxAvailability
+        self.currentAvailability = currentAvailability
+        self.requestType = requestType
+        self.thread = None
+
+    def to_json(self) -> str:
+        """Converts the object to a JSON string matching the Java structure"""
+        return json.dumps(
+            {
+                "version": self.version,
+                "serviceId": self.serviceId,
+                "taskId": self.taskId,
+                "maxAvailability": self.maxAvailability,
+                "currentAvailability": self.currentAvailability,
+                "requestType": self.requestType.value,  # Using .value for Enum serialization
+            },
+            indent=2,
+        )
+
+
+class BackpressureHandler:
+    # Track the number of messages currently being processed
+    current_parallel_executions = 0
+    # helps detect IDLE condition
+    last_data_message_time = 0
+    # helps prevent flooding the system with backpressure events
+    last_backpressure_event_time = 0
+    last_backpressure_event = ScalingRequestAlert.UPDATE
+
+    def __init__(
+        self,
+        channel: AbstractRobustChannel,
+        loop: AbstractEventLoop,
+        config: AMQConfiguration,
+    ):
+        self.lock = None
+        self.channel = channel
+        self.loop = loop
+        self.config = config
+        self.exchange = None
+        self.swarm_service_id = self.config.amq_adapter.swarm_service_id
+        self.swarm_task_id = self.config.amq_adapter.swarm_task_id
+        self.parallel_workers = self.config.backpressure.threshold_threads
+
+    def increase_parallel_executions(self):
+        """Increase the number of parallel executions"""
+        logging_info(
+            "Backpressure: Increase parallel executions, current=%s",
+            self.current_parallel_executions,
+        )
+        self.current_parallel_executions += 1
+
+    def decrease_parallel_executions(self):
+        """Decrease the number of parallel executions"""
+        logging_info(
+            "Backpressure: Decrease parallel executions, current=%s",
+            self.current_parallel_executions,
+        )
+        if self.current_parallel_executions > 0:
+            self.current_parallel_executions -= 1
+
+    def update_parallel_executions(self, count: int):
+        """Update the number of parallel executions"""
+        self.current_parallel_executions = count
+
+    def update_last_data_message_time(self):
+        logging_info("Backpressure: Update last data message time")
+        """Update the last data message time"""
+        self.last_data_message_time = time.time()
+
+    def start_backpressure_monitor(self) -> Thread:
+        # Start the Backpressure monitor loop
+        self.thread = Thread(target=self.backpressure_monitor_loop)
+        self.thread.start()
+        return self.thread
+
+    async def check_overload_condition(self):
+        """Check if the current parallel executions exceed the limit"""
+        self.update_last_data_message_time()
+        logging_info(
+            "Backpressure: Check overload condition, current=%s, max=%s",
+            self.current_parallel_executions,
+            self.parallel_workers,
+        )
+        if self.current_parallel_executions >= self.parallel_workers - 1:
+            # Check if the last backpressure event was not an overload
+            if self.last_backpressure_event != ScalingRequestAlert.OVERLOAD:
+                # Trigger the overload event
+                await self.handle_backpressure_overload_event()
+                self.last_backpressure_event_time = time.time()
+                self.last_backpressure_event = ScalingRequestAlert.OVERLOAD
+
+    def backpressure_monitor_loop(self):
+        _time_window_sec = self.config.backpressure.time_window
+        _idle_duration_millis = 1000 * self.config.backpressure.idle_duration
+        _loop = asyncio.new_event_loop()
+
+        async def _backpressure_monitor():
+            """Monitor the backpressure conditions and trigger events accordingly"""
+            while True:
+                logging_info(
+                    "Backpressure: Monitor loop, current=%s",
+                    self.current_parallel_executions,
+                )
+                logging_info(
+                    "Backpressure: Last data message time=%s, eventTime=%s",
+                    self.last_data_message_time,
+                    self.last_backpressure_event_time,
+                )
+                logging_info(
+                    "Backpressure: Last backpressure event=%s",
+                    self.last_backpressure_event,
+                )
+                # Check if the current time exceeds the last backpressure event time
+                if (
+                    self.current_parallel_executions >= self.parallel_workers
+                    and time.time() - self.last_backpressure_event_time
+                    > _time_window_sec
+                    and self.last_backpressure_event != ScalingRequestAlert.OVERLOAD
+                ):
+                    # Trigger the overload event
+                    await self.handle_backpressure_overload_event()
+                    self.last_backpressure_event_time = time.time()
+                    self.last_backpressure_event = ScalingRequestAlert.OVERLOAD
+                elif (
+                    self.current_parallel_executions == 0
+                    and time.time() - self.last_data_message_time
+                    >= _idle_duration_millis
+                    and self.last_backpressure_event != ScalingRequestAlert.IDLE
+                ):
+                    # Trigger the idle event
+                    await self._handle_backpressure_idle_event()
+                    self.last_backpressure_event = ScalingRequestAlert.IDLE
+                    self.last_backpressure_event_time = time.time()
+                else:
+                    # Trigger update event
+                    if (
+                        self.current_parallel_executions > 0
+                        and time.time() - self.last_backpressure_event_time
+                        > _time_window_sec
+                    ):
+                        # Trigger the update event
+                        _scaling_request: ScaleRequestV1 = ScaleRequestV1(
+                            self.swarm_service_id,
+                            self.swarm_task_id,
+                            self.parallel_workers,
+                            self.parallel_workers - self.current_parallel_executions,
+                            ScalingRequestAlert.UPDATE,
+                        )
+                        # Address the message to any adapter capable of supporting BACKPRESSURE request
+                        await self.publish_backpressure_request(_scaling_request)
+                        self.last_backpressure_event_time = time.time()
+                        self.last_backpressure_event = ScalingRequestAlert.UPDATE
+                await asyncio.sleep(_time_window_sec)
+
+        _loop.run_until_complete(_backpressure_monitor())
+
+    async def handle_backpressure_overload_event(self):
+        logging_warning("Backpressure: Capacity close to depleted!")
+        # Send an Overload event to the Management service
+        scaling_request: ScaleRequestV1 = ScaleRequestV1(
+            self.swarm_service_id,
+            self.swarm_task_id,
+            self.parallel_workers,
+            self.parallel_workers - self.current_parallel_executions,
+            ScalingRequestAlert.OVERLOAD,
+        )
+        # Address the message to any management service instance capable of supporting BACKPRESSURE request
+        await self.publish_backpressure_request(scaling_request)
+        # update / reset time-window so that the OVERLOAD is not sent too often
+        self.last_data_message_time = time.time()
+
+    async def _handle_backpressure_idle_event(self):
+        logging_warning("Backpressure: Service is idle.")
+        # Send an Idle event to the Management service
+        scaling_request: ScaleRequestV1 = ScaleRequestV1(
+            self.swarm_service_id,
+            self.swarm_task_id,
+            self.parallel_workers,
+            self.parallel_workers - self.current_parallel_executions,
+            ScalingRequestAlert.IDLE,
+        )
+        # Address the message to any adapter capable of supporting BACKPRESSURE request
+        await self.publish_backpressure_request(scaling_request)
+        # update / reset time-window so that the IDLE is not sent too often
+        self.last_data_message_time = time.time()
+
+    async def publish_backpressure_request(self, scaling_request: ScaleRequestV1):
+        # Publish the backpressure request to the management service
+        logging_info(
+            f"Publishing backpressure for {scaling_request.serviceId} with request type {scaling_request.requestType}"
+        )
+        if not self.exchange:
+
+            async def _wrap_rabbit_mq_api_init(channel):
+                _exchange = await channel.get_exchange(
+                    name="cleverthis.clevermicro.management"
+                )
+                return _exchange
+
+            self.exchange = await await_result(
+                asyncio.run_coroutine_threadsafe(
+                    _wrap_rabbit_mq_api_init(self.channel), self.loop
+                )
+            )
+
+        if self.exchange:
+
+            async def _wrap_rabbit_mq_api():
+                if not self.channel.is_closed:
+                    binary_content: bytes = scaling_request.to_json().encode("utf-8")
+                    pika_message: Message = Message(
+                        body=binary_content,
+                        content_encoding="utf-8",
+                        delivery_mode=2,
+                        content_type="application/octet-stream",
+                        headers=None,
+                        priority=0,
+                        correlation_id=None,
+                    )
+                    await self.exchange.publish(
+                        message=pika_message, routing_key="backpressure-scaling-v1"
+                    )
+                    logging_info(
+                        "Service Message Published to %s, msg: %s",
+                        self.exchange.name,
+                        str(binary_content),
+                    )
+                    return True
+                return False
+
+            await await_future(
+                asyncio.run_coroutine_threadsafe(_wrap_rabbit_mq_api(), self.loop)
+            )

amqp/adapter/cleverthis_service_adapter.py

@@ -0,0 +1,424 @@
+"""
+The layer that interfaces with the CleverThis service that this Adapter is integrating with.
+Methods here will invoke the actual CleverThis Service code, which are passed as Callable object.
+"""
+
+import asyncio
+import inspect
+import json
+from asyncio import AbstractEventLoop
+from typing import Dict, List, Callable, Coroutine, Any
+
+from aio_pika.abc import (
+    AbstractRobustChannel,
+    AbstractRobustConnection,
+    AbstractExchange,
+)
+from aiohttp import ClientSession, ClientResponse
+from dataclasses import dataclass
+from fastapi import HTTPException
+from starlette.responses import FileResponse
+
+from amqp.adapter.data_parser import AMQDataParser
+from amqp.adapter.data_response_factory import DataResponseFactory
+from amqp.adapter.file_handler import FileHandler, StreamingFileHandler
+from amqp.adapter.logging_utils import logging_error, logging_debug, logging_info
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import DataMessage, DataResponse
+from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
+from amqp.router.utils import await_result
+
+
+@dataclass
+class AMQMessage(DataMessage):
+    """
+    Expand the core DataMessage type to include the channel, which is required to create a dedicated queue
+    in case the response should initiate file download.
+    """
+
+    def __init__(self, data_message: DataMessage, connection: AbstractRobustConnection):
+        # Copy all fields from DataMessage
+        self.magic = data_message.magic
+        self.id = data_message.id
+        self.method = data_message.method
+        self.domain = data_message.domain
+        self.path = data_message.path
+        self.headers = data_message.headers
+        self.trace_info = data_message.trace_info
+        self.base64_body = data_message.base64_body
+        # Add the new field
+        self.connection = connection
+        self.extra_data = (
+            data_message.extra_data if hasattr(data_message, "extra_data") else {}
+        )
+
+
+async def _convert_file_response(
+    obj: FileResponse,
+    connection: AbstractRobustConnection,
+    file_handler: FileHandler,
+    loop: AbstractEventLoop | None,
+) -> str:
+    """
+    Converts a FileResponse object to a JSON string containing the filename and stream name.
+    The file content itself is transmitted through the named stream (which is implemented as temporary dedicated queue.
+    """
+
+    async def _wrap_amq_api_calls(
+        connection: AbstractRobustConnection,
+    ) -> tuple[AbstractRobustChannel, AbstractExchange, str]:
+        # Wrap calls to RabbitMQ API to make it Awaitable, so it can be run thread-safe with correct event loop
+        # This is needed because the RabbitMQ API is not thread-safe and needs to be called in the context of
+        # the event loop the first RabbitMQ connection was created with.
+        _channel = await connection.channel()
+        _exchange_name: str = AMQ_REPLY_TO_EXCHANGE
+        _exchange = await _channel.get_exchange(name=_exchange_name, ensure=True)
+        _queue = await _channel.declare_queue(
+            exclusive=False
+        )  # Declare a unique, non-exclusive queue
+        _queue_name = _queue.name
+        await _queue.bind(
+            exchange=_exchange_name,
+            routing_key=_queue_name,  # Use queue name as routing key
+        )
+        logging_debug(f"Publishing to exchange: {_exchange_name}, q: {_queue_name}")
+        return _channel, _exchange, _queue_name
+
+    # All calls to RabbitMQ API have to be called threadsafe, because those appear to be attached to parent
+    # event loop, and need to execute at that event loop, not the current one
+    # allow coroutine to execute. calling _future.result() will block the entire thread and the coroutine will never execute
+    _channel, _exchange, _routing_key = await await_result(
+        asyncio.run_coroutine_threadsafe(_wrap_amq_api_calls(connection), loop)
+    )
+    # publish_file also needs RabbitMQ API, ensure the loop is passed as argument
+    (_stream_name, _file_size) = await file_handler.publish_file(
+        _exchange, obj.path, _routing_key, loop
+    )
+    # as above, but luckily this time we don't need to wait for the result
+    _future = asyncio.run_coroutine_threadsafe(_channel.close(), loop)
+    return json.dumps(
+        {
+            "files": [
+                {
+                    "filename": obj.filename,
+                    "mediaType": "application/octet-stream",
+                    "streamName": _stream_name,
+                    "fileSize": _file_size,
+                }
+            ]
+        }
+    )
+
+
+def is_likely_json(text: str) -> bool:
+    """Quick test to see if should convert the response string to JSON format."""
+    _text = text.strip()
+    return (_text.startswith("{") and _text.endswith("}")) or (
+        _text.startswith("[") and _text.endswith("]")
+    )
+
+
+# =============  CleverThisServiceAdapter =============
+class CleverThisServiceAdapter:
+    """
+    A base class for CleverThis service adapter that handles the incoming AMQP messages and routes them to the
+    appropriate CleverThisService API endpoint.
+    IMPORTANT: This class is not intended to be used directly. It should be subclassed for each CleverThis service
+    and override the methods to handle the specific API calls.
+    """
+
+    def __init__(
+        self, amq_configuration: AMQConfiguration, session: ClientSession = None
+    ):
+        """
+        Initializes the CleverThisServiceAdapter with the given AMQ configuration and optional HTTP session.
+        The HTTP session is for the case when the adapter uses loose coupling with the CleverThis service via
+        HTTP REST API.
+        """
+        self.amq_configuration = amq_configuration
+        self.session = session
+        self.service_host = self.amq_configuration.dispatch.service_host
+        self.service_port = self.amq_configuration.dispatch.service_port
+        self.loop: AbstractEventLoop | None = None
+        self.require_authenticated_user = (
+            self.amq_configuration.amq_adapter.require_authenticated_user
+        )
+
+    def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
+        """
+        Pull the value of the Content-Type header from the message headers.
+        :param message_headers: The headers of the message.
+        return: The content type of the message.
+        """
+        for header, values in message_headers.items():
+            if header.lower() == "content-type":
+                return values[0].split(";")[0]
+        return "application/json"
+
+    async def get_current_user(self, token: str) -> object:
+        """
+        To Be Overriden in subclass for given CleverThis Service, to return the current user based on the token,
+        in format appropriate for the service.
+        :param token: The token to be used for authentication.
+        :return: A user object or None if the user is not authenticated.
+        """
+        return None
+
+    async def on_message(self, message: AMQMessage) -> DataResponse:
+        """
+        Called when a message is received from the AMQP.
+        :param message:
+        :return: AMQP response class with operation status code and optional error details.
+        """
+        _auth_user: Any | None = None
+        _amq_response: Any | None = None
+
+        logging_debug(
+            f"on_message: require_authenticated_user={self.require_authenticated_user}"
+        )
+        if self.require_authenticated_user:
+            _auth: str = message.headers.get("Authorization", "")
+            logging_debug(f"Auth: {_auth}")
+            if isinstance(_auth, List):
+                _auth = _auth[0]
+            if "Bearer" in _auth:
+                try:
+                    token = _auth.split(" ")[1]
+                    logging_info(f"Token: {token}")
+                    _auth_user = await self.get_current_user(token)
+                except Exception as error:
+                    logging_error(f"on_message: Error getting user: {error}")
+
+        if (
+            _auth_user
+            or not self.require_authenticated_user
+            or message.path.endswith("/login")
+        ):
+            method = message.method.lower()
+            if method == "get":
+                _amq_response = await self.get(message, _auth_user)
+            elif method == "put":
+                _amq_response = await self.put(message, _auth_user)
+            elif method == "post":
+                _amq_response = await self.post(message, _auth_user)
+            elif method == "head":
+                _amq_response = await self.head(message, _auth_user)
+            elif method == "delete":
+                _amq_response = await self.delete(message, _auth_user)
+            else:
+                raise ValueError(f"Unexpected HTTP method: {message.method}")
+        else:
+            _amq_response = DataResponseFactory.of(
+                message.id, 401, "text/plain", b"Unauthorized", message.trace_info
+            )
+        logging_info(
+            f"ALL DONE in on_message id:{message.id}, resp code:{_amq_response.response_code}"
+        )
+        return _amq_response
+
+    # ==================================================================================================
+    # =====================  C l e v e r T h i s    A P I    m e t h o d s  ============================
+    # ==================================================================================================
+
+    async def get(self, message: DataMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the GET requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
+        It will attempt to call the REST endpoint directly using HTTP protocol.
+        :param message: message from the AMQP that contains the GET request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+
+        # unmatched path - try to invoke the service directly on this path
+        url = f"http://{self.service_host}:{self.service_port}{message.path}"
+        headers = {**message.headers, **message.trace_info}
+        logging_info(
+            f"SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}"
+        )
+        if self.session:
+            _http_resp: ClientResponse = await self.session.get(url, headers=headers)
+            return DataResponseFactory.of(
+                message.id,
+                _http_resp.status,
+                _http_resp.content_type,
+                await _http_resp.read(),
+                message.trace_info,
+            )
+        return DataResponseFactory.of(
+            message.id,
+            404,
+            "text/plain",
+            str("Destination location does not exists").encode("utf-8"),
+            message.trace_info,
+        )
+
+    async def put(self, message: DataMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the PUT requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
+        It will attempt to call the REST endpoint directly using HTTP protocol.
+        :param message: message from the AMQP that contains the PUT request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        return await self.handle_possible_form(
+            message,
+            self.session.put(
+                f"http://{self.service_host}:{self.service_port}{message.path}",
+                data=message.body,
+                headers={**message.headers, **message.trace_info},
+            ),
+            auth_user,
+        )
+
+    async def post(self, message: DataMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the POST requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
+        It will attempt to call the REST endpoint directly using HTTP protocol.
+        :param message: message from the AMQP that contains the POST request, including body and headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        return await self.handle_possible_form(
+            message,
+            self.session.post(
+                f"http://{self.service_host}:{self.service_port}{message.path}",
+                data=message.body,
+                headers={**message.headers, **message.trace_info},
+            ),
+            auth_user,
+        )
+
+    async def handle_possible_form(
+        self, message: DataMessage, request_coroutine, auth_user: Any
+    ) -> DataResponse:
+        _args = AMQDataParser.parse_request_body(message)
+        return await request_coroutine
+
+    async def head(self, message: DataMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the HEAD requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
+        It will attempt to call the REST endpoint directly using HTTP protocol.
+        :param message: message from the AMQP that contains the POST request, including body and headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        raise NotImplementedError
+
+    async def delete(self, message: DataMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the DELETE requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
+        It will attempt to call the REST endpoint directly using HTTP protocol.
+        :param message: message from the AMQP that contains the POST request, including body and headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        raise NotImplementedError
+
+    @staticmethod
+    async def call_cleverthis_api(
+        message: AMQMessage,
+        args: Any,
+        api_method: Callable[[Any, Any], Coroutine],
+        success_code: int,
+        loop: AbstractEventLoop | None = None,
+    ) -> DataResponse:
+        """
+        Handles POST and PUT requests, which typically contain possibly complex payload data, passed as a dictionary.
+        Invokes the provided API method with the given arguments and authenticated user,
+        and returns the response as DataResponse.
+        :param message: DataMessage containing the details of the call and the payload
+        :param args: map of the arguments to pass to the API method
+        :param api_method: Callable - the method to invoke. Last parameter is the authenticated user
+        :param success_code: HTTP code to be returned when operation suceeds
+        :return: DataResponse class
+        """
+        try:
+            _verified_args = {}
+            _sig = inspect.signature(api_method)
+            _required_args = list(_sig.parameters.values())
+            # _required_args = getattr(api_method, '__annotations__', {})
+            for arg in _required_args:
+                if arg.name in args:
+                    # convert to another type if arg is not a str
+                    if arg.annotation == int:
+                        _verified_args[arg.name] = int(args[arg.name])
+                    elif arg.annotation == float:
+                        _verified_args[arg.name] = float(args[arg.name])
+                    elif arg.annotation == bool:
+                        _verified_args[arg.name] = bool(args[arg.name])
+                    # TODO: more?
+                    else:
+                        _verified_args[arg.name] = args[arg.name]
+                else:
+                    logging_info(
+                        f"{api_method.__name__}: Missing required argument: {arg.name}"
+                    )
+            logging_info(
+                f"INVOKE ENDPOINT: {api_method.__name__}, ARGS: {', '.join(f'{k}={v}' for k, v in _verified_args.items())}"
+            )
+            _result: Any = await api_method(**_verified_args)
+            logging_info(
+                f"ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}"
+            )
+
+            if isinstance(_result, FileResponse):
+                _json_result, _content_type = (
+                    await _convert_file_response(
+                        _result,
+                        message.connection,
+                        file_handler=StreamingFileHandler(),
+                        loop=loop,
+                    ),
+                    "application/octet-stream",
+                )
+            else:
+                _json_result = _result
+                _content_type = "application/json"
+
+            _binary_result = (
+                _json_result.encode("utf-8")
+                if hasattr(_json_result, "encode")
+                else (
+                    _json_result.body
+                    if hasattr(_json_result, "body")
+                    else (
+                        _json_result.read()
+                        if hasattr(_json_result, "read")
+                        else _json_result
+                    )
+                )
+            )
+            logging_info(
+                f"ALL DONE in call_cleverthis_api id:{message.id}, resp code:{success_code}"
+            )
+            return DataResponseFactory.of(
+                message.id,
+                success_code,
+                _content_type,
+                _binary_result,
+                message.trace_info,
+            )
+        except HTTPException as http_error:
+            _content = str(http_error.detail)
+            if not is_likely_json(_content):
+                _content = '{"error": "' + _content + '"}'
+            logging_error(f"Service REST endpoint invocation failed: {http_error}")
+            return DataResponseFactory.of(
+                message.id,
+                http_error.status_code,
+                "application/json",
+                _content.encode("utf-8"),
+                message.trace_info,
+            )
+        except Exception as error:
+            logging_error(f"Service endpoint invocation failed: {error}")
+            _content = str(error)
+            if not is_likely_json(_content):
+                _content = '{"error": "' + _content + '"}'
+            return DataResponseFactory.of(
+                message.id,
+                500,
+                "application/json",
+                _content.encode("utf-8"),
+                message.trace_info,
+            )

amqp/adapter/data_message_factory.py

@@ -0,0 +1,257 @@
+"""
+methods to create, serialize and deserialize DataMessage objects.
+"""
+
+import base64
+import json
+import re
+from io import BytesIO
+from datetime import datetime, timezone
+from typing import Dict, List
+
+import amqp.adapter.file_handler
+from amqp.model.model import DataMessage
+from amqp.model.snowflake_id import SnowflakeId
+from amqp.adapter.serializer import (
+    map_with_list_as_string,
+    map_as_string,
+    parse_map_list_string,
+    parse_map_string,
+)
+from amqp.router.utils import sanitize_as_rabbitmq_name
+from amqp.adapter.file_handler import StreamingFileHandler, FileHandler
+
+
+class DataMessageFactory:
+    SNOWFLAKE_ID_BITS = 80
+    SEQUENCE_BITS = 12
+    MACHINE_ID_BITS = 24
+    TIMESTAMP_BITS = SNOWFLAKE_ID_BITS - SEQUENCE_BITS - MACHINE_ID_BITS
+
+    MACHINE_ID_MASK = (1 << MACHINE_ID_BITS) - 1
+    SEQUENCE_MASK = (1 << SEQUENCE_BITS) - 1
+    MAGIC_BYTE_HTTP_REQUEST = "A"
+
+    _instance = None
+    snowflake_epoch = datetime(2024, 10, 1, 0, 0, tzinfo=timezone.utc).timestamp()
+
+    def __init__(self, machine_id: int):
+        self.machine_id = (machine_id & self.MACHINE_ID_MASK) << self.SEQUENCE_BITS
+        self.snowflake_sequence = 0
+        self.snowflake_last_timestamp = 0
+
+    @classmethod
+    def get_instance(cls, machine_id: int):
+        """
+        return reusable factory instance
+        :param machine_id: a generator ID, should be unique for each component to prevent duplicate IDs
+        :return: factory instance
+        """
+        if cls._instance is None:
+            cls._instance = DataMessageFactory(machine_id)
+        return cls._instance
+
+    def generate_snowflake_id(self) -> SnowflakeId:
+        """
+        Generate an unique ID following the SnowflakeID structure, see https://en.wikipedia.org/wiki/Snowflake_ID
+        :return: 80-bit wide ID
+        """
+        timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
+
+        if timestamp < self.snowflake_last_timestamp:
+            raise RuntimeError("Clock moved backwards. Refusing to generate id")
+
+        if timestamp == self.snowflake_last_timestamp:
+            self.snowflake_sequence = (self.snowflake_sequence + 1) & self.SEQUENCE_MASK
+            if self.snowflake_sequence == 0:
+                while timestamp == self.snowflake_last_timestamp:
+                    timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
+                self.snowflake_last_timestamp = timestamp
+        else:
+            self.snowflake_sequence = 0
+            self.snowflake_last_timestamp = timestamp
+
+        time = timestamp - 1000 * int(DataMessageFactory.snowflake_epoch)
+        return SnowflakeId(
+            time >> (64 - self.SEQUENCE_BITS - self.MACHINE_ID_BITS),
+            (
+                (time << (self.SEQUENCE_BITS + self.MACHINE_ID_BITS))
+                | self.machine_id
+                | self.snowflake_sequence
+            )
+            & 0xFFFFFFFFFFFFFFFF,
+        )
+
+    def create_request_message(
+        self,
+        method: str,
+        domain: str,
+        path: str,
+        headers: Dict[str, List[str]],
+        message,
+    ) -> DataMessage:
+        """
+        Encapsulate / hide the system level inputs, create the AMQ message from business relevant input
+        :param method: a method requested (e.g. HTTP method like GET, POST in case of REST request)
+        :param domain: service host name
+        :param path: path at the service
+        :param headers: HTTP headers
+        :param message: payload
+        :return: DataMessage object
+        """
+        serializable_headers = headers.copy() if headers else {}
+        payload = base64.b64encode(
+            message.encode("utf-8") if isinstance(message, str) else message
+        )
+        trace_info = {}
+        return DataMessage(
+            self.MAGIC_BYTE_HTTP_REQUEST,
+            self.generate_snowflake_id(),
+            method,
+            domain,
+            path,
+            serializable_headers,
+            trace_info,
+            payload,
+        )
+
+    @staticmethod
+    def amq_message_routing_key(message: DataMessage) -> str:
+        """
+        Constructs the message specific routing key.
+        :param message: DataMessage input
+        :return: routing key (as string) compliant to RabbitMQ allowed character set.
+        """
+        return sanitize_as_rabbitmq_name(f"{message.domain}.{message.path}")
+
+    @staticmethod
+    def get_timestamp_from_id(_id: SnowflakeId) -> int:
+        """
+        Helper method - retrieve the timestamp from the ID
+        :param _id: SnowflakeID (includes encoded timestamp)
+        :return: the message timestamp
+        """
+        bits = _id.get_bits()
+        low = bits[0] >> (
+            DataMessageFactory.SEQUENCE_BITS + DataMessageFactory.MACHINE_ID_BITS
+        )
+        high = bits[1] << (
+            64 - DataMessageFactory.SEQUENCE_BITS - DataMessageFactory.MACHINE_ID_BITS
+        )
+        return (high | low) + DataMessageFactory.snowflake_epoch * 1000
+
+    @staticmethod
+    def to_string(message: DataMessage) -> str:
+        """
+        Human-readable representation.
+        :param message: DataMessage input
+        :return: as string
+        """
+        return (
+            f"DataMessage{{"
+            f"id={message.id}, "
+            f"method='{message.method}', "
+            f"domain='{message.domain}', "
+            f"path='{message.path}', "
+            f"headers={{{map_with_list_as_string(message.headers)}}}, "
+            f"traceInfo={{{map_as_string(message.trace_info)}}}, "
+            f"base64body={message.base64_body.decode()}"
+            f"}}"
+        )
+
+    @staticmethod
+    def serialize(message: DataMessage) -> bytes:
+        """
+        Converts DataMessage to byte array
+        :param message: message to serialize
+        :return: messages as bytes
+        """
+        id_bytes = str(message.id).encode()
+        prolog = (
+            f"|m={message.method}|d={message.domain}|p={message.path}|h={{{map_with_list_as_string(message.headers)}}}|t={{{map_as_string(message.trace_info)}}}|b="
+        ).encode("utf-8")
+        header_length = 1 + len(id_bytes) + len(prolog)
+        prolog_len = header_length.to_bytes(2, byteorder="big")
+        msg_length = 2 + header_length + len(message.base64_body)
+        with BytesIO(bytearray(msg_length)) as baos:
+            baos.write(prolog_len)
+            baos.write(DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST.encode("utf-8"))
+            baos.write(id_bytes)
+            baos.write(prolog)
+            baos.write(message.base64_body)
+            return baos.getvalue()
+
+    @staticmethod
+    def from_bytes(input_bytes: bytes) -> DataMessage:
+        """
+        Builds the DataMessage from its serialized (byte array) form.
+        :param input_bytes: serialized message
+        :return: DataMessage instance
+        """
+        prolog_len = (input_bytes[0] << 8) + input_bytes[1]
+        metadata = input_bytes[2 : prolog_len + 2].decode()
+
+        pattern = re.compile(
+            f"{DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST}"
+            r"([0-9A-Fa-f]+)\|"
+            r"m=([^|]*)\|"
+            r"d=([^|]*)\|"
+            r"p=([^|]*)\|"
+            r"h=(\{[^}]*\})\|"
+            r"t=(\{[^}]*\})\|"
+            r"b="
+        )
+        match = pattern.match(metadata)
+        if not match:
+            raise ValueError(
+                f"DataMessage format does not match expected format. "
+                f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
+            )
+
+        id_str = match.group(1)
+        method = match.group(2)
+        domain = match.group(3)
+        path = match.group(4)
+        headers_str = match.group(5)
+        trace_info_str = match.group(6)
+        body_b64 = input_bytes[prolog_len + 2 :]
+
+        headers = parse_map_list_string(headers_str)
+        trace_info = parse_map_string(trace_info_str)
+
+        return DataMessage(
+            DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST,
+            SnowflakeId.from_hex(id_str),
+            method,
+            domain,
+            path,
+            headers,
+            trace_info,
+            body_b64,
+        )
+
+    @staticmethod
+    async def from_stream(
+        stream: bytes,
+        rabbitmq_url: str,
+        loop,
+        file_handler: FileHandler = StreamingFileHandler(),
+    ) -> DataMessage | None:
+        """
+        Builds the DataMessage from its serialized (byte array) form.
+        :param stream: serialized message
+        :param rabbitmq_url: RabbitMQ connection URL
+        :param loop: event loop
+        :param file_handler: file handler for downloading the message
+        :return: DataMessage instance
+        """
+        # async implementation
+        _message_data = json.loads(stream.decode())
+        _req_info = _message_data.get("req-info", "")
+        if _req_info:
+            (body, eof) = await file_handler.download_buffer(
+                loop=loop, rabbitmq_url=rabbitmq_url, queue_name=_req_info
+            )
+            if eof == amqp.adapter.file_handler.END_OF_FILE:
+                return DataMessageFactory.from_bytes(body)
+        return None

amqp/adapter/data_parser.py

@@ -0,0 +1,201 @@
+import io
+import json
+import python_multipart
+
+from fastapi import UploadFile
+from python_multipart.multipart import Field, File
+from typing import Tuple, Dict
+from urllib.parse import parse_qs, urlparse
+from xml.etree import ElementTree
+
+from amqp.adapter.logging_utils import logging_error, logging_debug
+from amqp.model.model import DataMessage
+
+
+class MultipartFormDataParser:
+    """
+    Parses multipart/form-data content and extracts form fields (key/value pairs) and files.
+    Note: multipart message that contains file(s) to upload are converted to CleverMicro JSON-based streaming format,
+    where files are transmitted separately via stream using dedicated queue, so the message content is no longer
+    the HTTP multipart format, but the Content-Type is being preserved for compatibility with the original message.
+    This class supports both formats (multipart and JSON) depending on the message content, because if no file is
+    included, the message is transmitted in the unchanged multipart/form-data format.
+    """
+
+    def __init__(self, message: DataMessage):
+        self.form_data: dict = {}
+        self.is_multipart = False
+        self.is_json = False
+        self.is_valid = True
+        # Convert each list of strings to a single string joined by newline, then to bytes
+        _headers = {
+            key: "\n".join(value).encode("utf-8")
+            for key, value in message.headers.items()
+        }
+        try:
+            if (
+                hasattr(message, "extra_data")
+                and isinstance(message.extra_data, dict)
+                and len(message.extra_data) > 0
+            ):
+                try:
+                    self.form_data = json.loads(message.body())
+                    self.is_json = True
+                    if self.form_data["files"] and self.form_data["form"]:
+                        self.form_data = AMQDataParser.process_form_data(self.form_data)
+                except Exception as jsonerror:
+                    logging_error(f"Parsing JSON: {jsonerror}")
+                    self.is_valid = False
+            else:
+                python_multipart.parse_form(
+                    _headers, io.BytesIO(message.body()), self.on_field, self.on_file
+                )
+                self.is_multipart = True
+        except Exception as e:
+            logging_error(
+                f"Error parsing multipart/form-data: {e}. Trying parsing as JSON."
+            )
+            try:
+                self.form_data = json.loads(message.body())
+                self.is_json = True
+                if self.form_data["files"] and self.form_data["form"]:
+                    self.form_data = AMQDataParser.process_form_data(self.form_data)
+            except Exception as jsonerror:
+                logging_error(f"Parsing JSON: {jsonerror}")
+                self.is_valid = False
+
+    def on_field(self, field: Field):
+        logging_debug(str(field))
+        self.form_data[field.field_name.decode("utf-8")] = field.value
+
+    def on_file(self, file: File):
+        logging_debug(str(file))
+        self.form_data[file.field_name.decode("utf-8")] = UploadFile(
+            file.file_object, size=file.size, filename=file.file_name
+        )
+
+
+class AMQDataParser:
+    """
+    A class for parsing AMQ data messages. It provides methods to parse form data,
+    """
+
+    @staticmethod
+    def process_form_data(json_input) -> dict:
+        """
+        Processes the input JSON to extract form data. The JSON has two top entries: 'files' and 'form'.
+        This is part of the file stream implementation in AMQ adapter to stream large files in chunks.
+        'files' contains information about the AMQ queue from which read the file, while 'form' contains
+        additional non-file form values.
+        This function combines the file and values at the top level in the returned dictionary.
+        Args:
+            json_input: JSON string or dict containing the form and files data
+
+        Returns:
+            dict: Processed form data with files data overriding non-empty values
+        """
+        # Load JSON if input is string
+        if isinstance(json_input, str):
+            data = json.loads(json_input)
+        else:
+            data = json_input
+
+        result = {}
+
+        # Process form data - take first element of each list
+        for key, value in data["form"].items():
+            if value:  # Only process if list is not empty
+                result[key] = value[0]
+
+        # Override with files data where available
+        for key, value in data["files"].items():
+            if value:  # Only override if files data is non-empty
+                result[key] = value[0]
+
+        return result
+
+    @staticmethod
+    def parse_get_url(url: str) -> Tuple[str, Dict[str, str]]:
+        """
+        Parses an HTTP GET URL and returns (context_path, query_params_dict)
+
+        Args:
+            url: The URL string to parse (e.g., "http://example.com/path?foo=1&bar=2&foo=3")
+
+        Returns:
+            tuple: (context_path, query_params_dict)
+            - context_path: The path part of the URL (e.g., "/path")
+            - query_params_dict: Query parameters where values are str for single value or list of str for multivalue
+        """
+        _parsed = urlparse(url)
+        _query_params = parse_qs(_parsed.query)
+        # Convert values from lists to single values when there's only one value
+        _simplified_params = {
+            k: v[0] if len(v) == 1 else v for k, v in _query_params.items()
+        }
+        return _parsed.path, _simplified_params
+
+    @staticmethod
+    def parse_request_body(message: DataMessage):
+        """
+        Parses the HTTP POST request body based on the MIME type.
+        This is typically called fom the concrete CleverThis Service to extract parameters from the message body.
+        Args:
+            message (DataMessage): Contains raw request and MIME headers.
+
+        Returns:
+            dict: Parsed data as a dictionary.
+        """
+        _content_type = message.headers.get("Content-Type", "")
+        if not _content_type:
+            raise ValueError("Content-Type header is required")
+        content_type = _content_type[0]
+        # Parse JSON
+        if content_type == "application/json":
+            try:
+                return json.loads(message.body())
+            except json.JSONDecodeError as e:
+                logging_error(f"Invalid JSON: {e}")
+                raise ValueError(f"Invalid JSON: {e}")
+
+        # Parse URL-encoded form data
+        elif content_type == "application/x-www-form-urlencoded":
+            try:
+                _form_data: dict = {
+                    k: v[0] if len(v) == 1 else v
+                    for k, v in parse_qs(message.body()).items()
+                }
+                if len(_form_data) == 0 and len(message.body()) > 0:
+                    _form_data = json.loads(message.body())
+                    if _form_data["files"] and _form_data["form"]:
+                        _form_data = MultipartFormDataParser.process_form_data(
+                            _form_data
+                        )
+                return _form_data
+            except Exception as e:
+                logging_error(f"Invalid URL-encoded form data: {e}")
+                raise ValueError(f"Invalid URL-encoded form data: {e}")
+
+        # Parse multipart/form-data
+        elif content_type.startswith("multipart/form-data"):
+            try:
+                parser: MultipartFormDataParser = MultipartFormDataParser(message)
+                return parser.form_data
+            except Exception as e:
+                logging_error(f"Invalid multipart/form-data: {e}")
+                raise ValueError(f"Invalid multipart/form-data: {e}")
+
+        # Parse plain text
+        elif content_type == "text/plain":
+            return {"text": message.body()}
+
+        # Parse XML
+        elif content_type == "application/xml":
+            try:
+                _root = ElementTree.fromstring(message.body())
+                return {elem.tag: elem.text for elem in _root}
+            except ElementTree.ParseError as e:
+                raise ValueError(f"Invalid XML: {e}")
+
+        else:
+            raise ValueError(f"Unsupported Content-Type: {content_type}")

amqp/adapter/data_response_factory.py

@@ -0,0 +1,125 @@
+"""
+method to create, serialize and deserialize DataResponse message
+"""
+
+import base64
+import re
+from io import BytesIO
+from amqp.model.model import DataResponse
+from amqp.model.snowflake_id import SnowflakeId
+from amqp.adapter.serializer import parse_map_string, map_as_string
+
+
+class DataResponseFactory:
+    MAGIC_BYTE_HTTP_REQUEST = "A"
+
+    @staticmethod
+    def create_async_response_message(_id: SnowflakeId) -> DataResponse:
+        """
+        Helper method to create standard OK response.
+        :param _id: ID
+        :return: Response message
+        """
+        return DataResponse(
+            str(_id), 200, "application/text", "OK".encode("utf-8"), "", "", {}
+        )
+
+    @staticmethod
+    def of(
+        id: SnowflakeId,
+        response_code: int,
+        content_type: str,
+        body: bytes,
+        trace_info: dict[str, str],
+    ) -> DataResponse:
+        """
+        Create the DataResponse message supplying all key values
+        :param id: SnowflakeID
+        :param response_code: HTTP response code
+        :param content_type: MIME content type
+        :param body: payload
+        :param trace_info: Jaeger trace info (id)
+        :return: DataResponseMessage object
+        """
+        return DataResponse(
+            id=id.as_string(),
+            response_code=response_code,
+            content_type=content_type,
+            response=body,
+            error=None,
+            error_cause=None,
+            trace_info=trace_info,
+        )
+
+    @staticmethod
+    def serialize(response: DataResponse) -> bytes:
+        """
+        Serialize the message to byte array (to be sent over RabbitMQ)
+        :param response: Object to serialize
+        :return: byte array as serialized message
+        """
+        id_bytes = response.id.encode("utf-8")
+        prolog = (
+            f"|r={response.response_code}|c={response.content_type}|e={response.error}|f={response.error_cause}|t={{{map_as_string(response.trace_info)}}}|b="
+        ).encode("utf-8")
+        header_length = 1 + len(id_bytes) + len(prolog)
+        prolog_len = header_length.to_bytes(2, byteorder="big")
+        msg_length = 2 + header_length + len(response.response)
+        with BytesIO(bytearray(msg_length)) as baos:
+            baos.write(prolog_len)
+            baos.write(DataResponseFactory.MAGIC_BYTE_HTTP_REQUEST.encode("utf-8"))
+            baos.write(id_bytes)
+            baos.write(prolog)
+            baos.write(base64.b64encode(response.response))
+            return baos.getvalue()
+
+    @staticmethod
+    def from_bytes(input_bytes: bytes) -> DataResponse:
+        """
+        Buil;ds the DataResponse object from its serialized form
+        :param input_bytes: byte array - serialized response
+        :return: DataResponse object
+        """
+        prolog_len = (input_bytes[0] << 8) + input_bytes[1]
+        metadata = input_bytes[2 : prolog_len + 2].decode()
+
+        pattern = re.compile(
+            f"{DataResponseFactory.MAGIC_BYTE_HTTP_REQUEST}"
+            r"([0-9A-Fa-f]+)\|"
+            r"r=([^|]*)\|"
+            r"c=([^|]*)\|"
+            r"e=([^|]*)\|"
+            r"f=([^|]*)\|"
+            r"t=(\{[^}]*\})\|"
+            r"b="
+        )
+        match = pattern.match(metadata)
+        if not match:
+            raise ValueError(
+                f"DataResponse format does not match expected format. "
+                f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
+            )
+
+        id_str = match.group(1)
+        response_code = int(match.group(2))
+        content_type = match.group(3)
+        error = match.group(4)
+        error_cause = match.group(5)
+        trace_info_str = match.group(6)
+        body_b64 = input_bytes[prolog_len + 2 :]
+
+        trace_info = parse_map_string(trace_info_str)
+
+        return DataResponse(
+            id_str,
+            response_code,
+            content_type,
+            base64.b64decode(body_b64),
+            error,
+            error_cause,
+            trace_info,
+        )
+
+
+# Ensure backward compatibility
+AMQResponseFactory = DataResponseFactory

amqp/adapter/file_handler.py

@@ -0,0 +1,409 @@
+"""
+implements the file downloader for the amqp adapter, where file is sent in chunks over RabbitMQ
+"""
+
+import asyncio
+import json
+import os
+import logging
+from asyncio import Future, AbstractEventLoop
+from collections import defaultdict
+from pathlib import Path
+
+from aio_pika import connect_robust, Message
+from aio_pika.abc import (
+    AbstractIncomingMessage,
+    AbstractRobustChannel,
+    AbstractQueue,
+    AbstractExchange,
+    DeliveryMode,
+)
+
+from amqp.adapter.logging_utils import logging_error, logging_debug, logging_info
+from amqp.router.utils import await_result
+
+# Global dictionary to track completed downloads
+download_locations = defaultdict(str)
+IN_PROGRESS = 0
+END_OF_FILE = 1
+CANCELLED = 2
+
+
+class FileHandler:
+    """
+    The abstract base class for file handling. This is mainly to make the code testable, as there are only 2
+    options for pushing (uploading) a file(s) to a service: as HTTP multipart message delivered via RabbitMQ
+    or via stream in RabbitMQ. Downloading a file is always done via RabbitMQ stream. Thus the method signiture
+    is tighly coupled to the RabbitMQ API in v1. Only alternative implementation is a mock object in unittests.
+    """
+
+    def __init__(self):
+        pass
+
+    def ensure_directory_exists(self, filepath) -> str:
+        """
+        Ensures that the directory for the given filepath exists, and handles file versioning.
+
+        Args:
+            filepath: The path to the file.
+
+        Returns:
+            The original filepath if the file does not exist, or a modified filepath
+            pointing to the next available version of the file if it does.
+        """
+        # 1. Ensure directory exists
+        directory = os.path.dirname(filepath)
+        if directory and not os.path.exists(directory):
+            os.makedirs(directory)
+
+        # 2. Handle file versioning
+        if not os.path.exists(filepath):
+            return filepath  # File doesn't exist, return original path
+
+        # File exists, find next available version
+        base, ext = os.path.splitext(filepath)
+        version = 1
+        while True:
+            new_filepath = f"{base}.{version}{ext}"
+            if not os.path.exists(new_filepath):
+                return new_filepath  # Found a free version, return this path
+            version += 1
+
+    async def download_buffer(self, loop, rabbitmq_url, queue_name) -> (bytearray, int):
+        """
+        Asynchronously downloads a bytearray from RabbitMQ based on the provided file information.
+        It assumes entire array is in the first (and only) chunk
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            queue_name (str): The directory where the file should be saved.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def download_file(
+        self, loop, rabbitmq_url, file_info, message_data, output_dir
+    ) -> (int, int):
+        """
+        Asynchronously downloads a file from RabbitMQ based on the provided file information.
+        takes the file_info and message_data and output_dir, creates queue consumer and waits for the file
+        to be downloaded. File may consist of multiple chunks, so the function will wait until all chunks are received.
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            file_info (dict): A dictionary containing file metadata
+            message_data (dict): The entire message
+            output_dir (str): The directory where the file should be saved.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def on_message(
+        self,
+        message: AbstractIncomingMessage,
+        json_data: bytes,
+        rabbitmq_url: str,
+        loop,
+        output_dir="downloaded_files",
+    ) -> defaultdict:
+        """
+        Asynchronous callback function for handling messages from the 'amq-cleverswarm' queue.
+
+        Args:
+            message: The incoming RabbitMQ message.
+            rabbitmq_url: The RabbitMQ connection URL.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def publish_file(
+        self,
+        exchange: AbstractExchange,
+        file_path: Path,
+        routing_key: str,
+        loop: AbstractEventLoop,
+        max_chunk_size: int = 2**20,  # 1MB default chunk size
+        content_type: str = "application/octet-stream",  # Default content type
+        delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
+    ) -> tuple[str, int]:
+        """
+        Reads a file from the file system and publishes its content to a RabbitMQ exchange
+        in chunks.  It declares a queue with a random name, binds it to the specified
+        exchange, and uses the queue name as the routing key.
+
+        Args:
+            exchange: The aio_pika RobustChannel to use for communication with RabbitMQ.
+            file_path: The path to the file to publish.
+            max_chunk_size: The maximum size of each chunk (in bytes).
+            routing_key: The name of the RabbitMQ exchange routing key to publish to correct queue.
+            loop: correct event loop
+            max_chunk_size: chunk size to send
+            content_type: The content type of the file data.
+            delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT).
+
+        Returns:
+            The name of the randomly generated queue where the file content was published.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+
+# The StreamingFileHandler class is a subclass of FileHandler that implements the download_file method
+class StreamingFileHandler(FileHandler):
+    """
+    Handles file downloads from RabbitMQ streams.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    async def download_file(
+        self, loop, rabbitmq_url, file_info, message_data, output_dir
+    ) -> (int, int):
+        """
+        Asynchronously downloads a file from RabbitMQ based on the provided file information.
+        takes the file_info and message_data and output_dir, creates queue consumer and waits for the file to be downloaded
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            file_info (dict): A dictionary containing file metadata
+            message_data (dict): The entire message
+            output_dir (str): The directory where the file should be saved.
+        """
+        global download_locations
+        filename = file_info["filename"]
+        size = file_info["size"]
+        stream_name = file_info["streamName"]
+        queue_name = file_info["queue_name"]
+        filepath = os.path.join(output_dir, filename)
+        filepath = self.ensure_directory_exists(
+            filepath
+        )  # if filename exists, create a new (next) version
+        download_locations[file_info["key"]] = filepath
+        filename = os.path.basename(
+            filepath
+        )  # if a new version was created, ensure to use that new version
+        file_info["filename"] = filename
+
+        logging_debug(
+            f"Downloading file: {filename} (size: {size} bytes) from stream: {stream_name}:{queue_name} to {filepath}"
+        )
+
+        async def wrap_rabbit_mq_calls() -> tuple[int, int] | None:
+            connection = await connect_robust(rabbitmq_url, loop=loop)
+            try:
+                _file_size = 0
+                _eof = IN_PROGRESS
+                channel: AbstractRobustChannel = await connection.channel()
+                queue: AbstractQueue = await channel.get_queue(queue_name, ensure=False)
+                logging_debug(
+                    f"Awaiting all file chunks being downloaded for {filename}"
+                )
+                async with queue.iterator() as queue_iter:
+                    async for message in queue_iter:
+                        async with message.process():
+                            with open(filepath, "ab") as f:
+                                f.write(message.body)
+                            _file_size = _file_size + message.body_size
+                            _eof = int(message.headers.get("eof", END_OF_FILE))
+                            if _eof != IN_PROGRESS:
+                                break
+            except Exception as e:
+                logging_error(f"File download failed: {e}")
+            finally:
+                await connection.close()
+
+            return _file_size, _eof
+
+        if queue_name:
+            _local_file_size, _local_eof = await await_result(
+                asyncio.run_coroutine_threadsafe(wrap_rabbit_mq_calls(), loop=loop)
+            )
+            return _local_file_size, _local_eof
+
+        return 0, CANCELLED
+
+    async def download_buffer(self, loop, rabbitmq_url, queue_name) -> (bytearray, int):
+        """
+        Asynchronously downloads a bytearay from RabbitMQ based on the provided file information.
+        It assumes entire array is in the first (and only) chunk
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            queue_name (str): The directory where the file should be saved.
+        """
+        logging_debug(f"Downloading buffer from stream: {queue_name}")
+        if queue_name:
+
+            async def wrap_rabbit_mq_calls() -> tuple[bytearray, int]:
+                _connection = await connect_robust(rabbitmq_url, loop=loop)
+                _eof = IN_PROGRESS
+                _res = bytearray()
+                try:
+                    channel: AbstractRobustChannel = await _connection.channel()
+                    queue: AbstractQueue = await channel.get_queue(
+                        queue_name, ensure=False
+                    )
+                    logging_debug(
+                        f"Awaiting all buffer chunks being downloaded for {queue_name}"
+                    )
+                    async with queue.iterator() as queue_iter:
+                        async for message in queue_iter:
+                            async with message.process():
+                                _res.extend(message.body)
+                                _eof = message.headers.get("eof", END_OF_FILE)
+                                if _eof != IN_PROGRESS:
+                                    break
+                    logging_info(f"Finished downloading buffer {queue_name}")
+                except Exception as e:
+                    logging_error(f"Downloading buffer: {e}")
+
+                await _connection.close()
+                return _res, _eof
+
+            _local_res, _local_eof = await await_result(
+                asyncio.run_coroutine_threadsafe(wrap_rabbit_mq_calls(), loop=loop)
+            )
+            return _local_res, _local_eof
+        return 0, CANCELLED
+
+    async def on_message(
+        self,
+        message: AbstractIncomingMessage,
+        json_data: bytes,
+        rabbitmq_url: str,
+        loop,
+        output_dir="downloaded_files",
+    ) -> defaultdict:
+        """
+        Asynchronous callback function for handling messages from the 'amq-cleverswarm' queue.
+
+        Args:
+            message: The incoming RabbitMQ message.
+            json_data: The JSON data from the message.
+            rabbitmq_url: The RabbitMQ connection URL.
+            loop: The asyncio event loop of the RabbitMQ API
+            output_dir
+        """
+        global download_locations
+        try:
+            _message_data = json.loads(message.body.decode())
+            _amq_message_data = json.loads(json_data.decode())
+
+            files_to_download = []
+            for _key, _file_type in _amq_message_data["files"].items():
+                if len(_file_type) > 0:
+                    files_info = _file_type[0]
+                    stream_name = files_info["streamName"]
+                    files_info["queue_name"] = (
+                        _message_data.get(stream_name, "") if stream_name else ""
+                    )
+                    files_info["key"] = _key
+                    files_to_download.append(files_info)
+
+            if not files_to_download:
+                logging_info(
+                    " [%s] No files to download in this message.", message.delivery_tag
+                )
+                # await message.ack()
+                return defaultdict()
+
+            # Create a task for each file download
+            _tasks = []
+            for file_info in files_to_download:
+                logging_info(
+                    f" [{message.delivery_tag}] about to create task for {file_info}"
+                )
+                _task = asyncio.create_task(
+                    self.download_file(
+                        loop, rabbitmq_url, file_info, _message_data, output_dir
+                    )
+                )
+                _tasks.append(_task)
+            logging_info(
+                f" [{message.delivery_tag}] Waiting for all files to download...gather(*tasks)"
+            )
+            await asyncio.gather(*_tasks)  # Wait for all downloads to complete
+
+            logging_info(f"All files downloaded. d-tag: {message.delivery_tag}")
+        except Exception as e:
+            logging_error(f"Building DataMessage: {e}")
+            asyncio.run_coroutine_threadsafe(
+                message.ack(), loop=loop
+            )  # ACK -> ignore the message (do not redeliver)
+
+        return download_locations
+
+    async def publish_file(
+        self,
+        exchange: AbstractExchange,
+        file_path: Path,
+        routing_key: str,
+        loop: AbstractEventLoop,
+        max_chunk_size: int = 2**20,  # 1MB default chunk size
+        content_type: str = "application/octet-stream",  # Default content type
+        delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
+    ) -> tuple[str, int]:
+        """
+        Reads a file from the file system and publishes its content to a RabbitMQ exchange
+        in chunks.  It declares a queue with a random name, binds it to the specified
+        exchange, and uses the queue name as the routing key.
+
+        Args:
+            exchange: The aio_pika RobustChannel to use for communication with RabbitMQ.
+            file_path: The path to the file to publish.
+            max_chunk_size: The maximum size of each chunk (in bytes).
+            routing_key: The name of the RabbitMQ exchange routing key to publish to correct queue.
+            loop: correct event loop
+            max_chunk_size: chunk size to send
+            content_type: The content type of the file data.
+            delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT).
+
+        Returns:
+            The name of the randomly generated queue where the file content was published.
+        """
+        if not file_path.is_file():
+            raise FileNotFoundError(f"File not found: {file_path}")
+        _file_size = os.path.getsize(file_path)
+
+        with open(file_path, "rb") as f:
+            _offset = 0
+            while True:
+                _chunk = f.read(max_chunk_size)
+                if not _chunk:
+                    break  # End of file
+
+                message = Message(
+                    body=_chunk,
+                    content_type=content_type,
+                    delivery_mode=delivery_mode,
+                    headers={
+                        "file_name": file_path.name,
+                        "pos": _offset,
+                        "total_size": _file_size,
+                        "chunk_size": len(_chunk),
+                        "eof": (
+                            END_OF_FILE
+                            if max_chunk_size + _offset >= _file_size
+                            else IN_PROGRESS
+                        ),
+                    },
+                )
+                logging.debug(
+                    f"Going to Publish chunk {_offset}/{_file_size} to {exchange.name}:{routing_key}"
+                )
+                _future = asyncio.run_coroutine_threadsafe(
+                    exchange.publish(message, routing_key=routing_key), loop
+                )
+                while not _future.done():
+                    # sleep allows coroutine to execute. must wait for completion to ensure all chunks written in right order
+                    await asyncio.sleep(0.02)
+                    logging_debug(
+                        f"Waiting for chunk {_offset}/{_file_size} future= {_future}"
+                    )
+
+                _offset += len(_chunk)
+                logging_debug(
+                    f"Published chunk {_offset}/{_file_size} to {exchange.name}:{routing_key}"
+                )
+
+        logging_debug(
+            f"File {file_path} published to RabbitMQ exchange {exchange.name}, queue {routing_key}"
+        )
+        return routing_key, _file_size

amqp/adapter/http_adapter.py

@@ -1,68 +0,0 @@
-import logging
-from typing import Dict, List
-from aiohttp import ClientSession, ClientResponse
-
-from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import AMQMessage
-
-
-class HttpAdapter:
-    def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession):
-        self.amq_configuration = amq_configuration
-        self.session = session
-        self.service_host = self.amq_configuration.dispatch.service_host
-        self.service_port = self.amq_configuration.dispatch.service_port
-
-    async def handle(self, message: AMQMessage) -> ClientResponse:
-        method = message.method.lower()
-        if method == 'get':
-            return await self.get(message)
-        elif method == 'put':
-            return await self.put(message)
-        elif method == 'post':
-            return await self.post(message)
-        elif method == 'head':
-            return await self.head(message)
-        elif method == 'delete':
-            return await self.delete(message)
-        else:
-            raise ValueError(f"Unexpected HTTP method: {message.method}")
-
-    async def get(self, message: AMQMessage) -> ClientResponse:
-        url = f"http://{self.service_host}:{self.service_port}{message.path}"
-        headers = {**message.headers, **message.trace_info}
-        logging.info(f" [*] SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}")
-        return await self.session.get(url, headers=headers)
-
-    async def put(self, message: AMQMessage) -> ClientResponse:
-        return await self.handle_possible_form(
-            self.session.put(
-                f"http://{self.service_host}:{self.service_port}{message.path}",
-                data=message.body,
-                headers={**message.headers, **message.trace_info},
-            )
-        )
-
-    async def post(self, message: AMQMessage) -> ClientResponse:
-        return await self.handle_possible_form(
-            self.session.post(
-                f"http://{self.service_host}:{self.service_port}{message.path}",
-                data=message.body,
-                headers={**message.headers, **message.trace_info},
-            )
-        )
-
-    async def handle_possible_form(self, request_coroutine) -> ClientResponse:
-        return await request_coroutine
-
-    async def head(self, message: AMQMessage) -> ClientResponse:
-        raise NotImplementedError
-
-    async def delete(self, message: AMQMessage) -> ClientResponse:
-        raise NotImplementedError
-
-    def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
-        for header, values in message_headers.items():
-            if header.lower() == 'content-type':
-                return values[0].split(';')[0]
-        return 'application/json'

amqp/adapter/logging_utils.py

@@ -0,0 +1,125 @@
+import inspect
+import logging
+import os
+import sys
+import threading
+import traceback
+
+verbose = os.environ.get("AMQ_VERBOSE_LOGGING", "0") == "1"
+
+
+def get_context_info():
+    """
+    Retrieves the current module, line number, and function name.
+
+    Returns:
+        A tuple containing (module name, line number, function name).
+        Returns (None, None, None) if it fails to retrieve the information.
+    """
+    try:
+        # Get the frame object for the current function call
+        frame = inspect.currentframe()
+        if frame is not None:
+            # Get the frame object for the caller of the current function
+            frame = frame.f_back.f_back
+        if frame:
+            code_context = frame.f_code
+            module_name = code_context.co_filename
+            module_name = (
+                "amqp" + module_name.split("/amqp", 1)[-1]
+                if "/amqp" in module_name
+                else module_name
+            )
+            line_number = frame.f_lineno
+            function_name = code_context.co_name
+            return module_name, line_number, function_name
+        else:
+            return None, None, None
+    except Exception:
+        return None, None, None
+
+
+def logging_error(msg: str, *args) -> None:
+    """
+    Log an error message according to current logging for 'ERROR' level.
+    Add module name, line and function name where the error occurred, on single line.
+    Args:
+        msg (str): The error message to log.
+    """
+    _exec_info = sys.exc_info()[2]
+    _thread_id = threading.get_ident()
+    if _exec_info:
+        _tb = traceback.extract_tb(sys.exc_info()[2])[-1]
+        logging.error(
+            f"{_thread_id} [E] '{msg}' ---> '{_tb.filename}':{_tb.lineno}, '{_tb.name}()'",
+            *args,
+        )
+    else:
+        module, line_number, function_name = get_context_info()
+        if module and line_number and function_name:
+            logging.error(
+                f"{_thread_id} [E] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+                *args,
+            )
+        else:
+            logging.error(f"{_thread_id} [E] {msg}", *args)
+
+
+def logging_warning(msg: str, *args) -> None:
+    """
+    Log a warning message according to current logging for 'WARN' level.
+    Add module name, line and function name where the print occurred, on single line.
+    Args:
+        msg (str): The warning message to log.
+    """
+    _thread_id = threading.get_ident()
+    if not verbose:
+        logging.warning(f"{_thread_id} [W] {msg}", *args)
+        return
+    module, line_number, function_name = get_context_info()
+    if module and line_number and function_name:
+        logging.warning(
+            f"{_thread_id} [W] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+            *args,
+        )
+    else:
+        logging.warning(f"{_thread_id} [W] {msg}", *args)
+
+
+def logging_info(msg: str, *args) -> None:
+    """
+    Log an info message according to current logging for 'INFO' level.
+    Add module name, line and function name where the print occurred, on single line.
+    Args:
+        msg (str): The info message to log.
+    """
+    _thread_id = threading.get_ident()
+    if not verbose:
+        logging.info(f"{_thread_id} [*] {msg}", *args)
+        return
+    module, line_number, function_name = get_context_info()
+    if module and line_number and function_name:
+        logging.info(
+            f"{_thread_id} [*] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+            *args,
+        )
+    else:
+        logging.info(f"{_thread_id} [*] {msg}", *args)
+
+
+def logging_debug(msg: str, *args) -> None:
+    """
+    Log a debug message according to current logging for 'DEBUG' level.
+    Add module name, line and function name where the print occurred, on single line.
+    Args:
+        msg (str): The debug message to log.
+    """
+    _thread_id = threading.get_ident()
+    module, line_number, function_name = get_context_info()
+    if module and line_number and function_name:
+        logging.info(
+            f"{_thread_id} [DBG] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+            *args,
+        )
+    else:
+        logging.debug(f"{_thread_id} [DBG] {msg}", *args)

amqp/adapter/pydantic_serializer.py

@@ -0,0 +1,172 @@
+"""
+This module provides serialization and deserialization functions for Pydantic models that extend BaseModel
+"""
+
+import uuid
+import json
+from datetime import datetime
+from typing import Dict, Any
+from pydantic import BaseModel
+from urllib.parse import urlparse, parse_qs
+
+from amqp.adapter.logging_utils import logging_debug
+
+
+def serialize_object(obj: BaseModel) -> str:
+    """
+    Serializes an R2RSerializable object to a string in the format
+    "<ClassName>:{key1:value1, key2:value2,...}".  Handles UUIDs and datetimes,
+    and nested Pydantic models.
+
+    Args:
+        obj: The BaseModel object to serialize.
+
+    Returns:
+        A string representation of the object.
+    """
+    class_name = obj.__class__.__name__
+    obj_dict = {}
+    for key, value in obj.model_dump().items():
+        if isinstance(value, uuid.UUID):
+            obj_dict[key] = str(value)
+        elif isinstance(value, datetime):
+            obj_dict[key] = value.isoformat()
+        elif isinstance(value, dict):  # Ensure dictionaries are handled
+            obj_dict[key] = value
+        elif isinstance(value, BaseModel):
+            obj_dict[key] = serialize_object(value)  # Recursively serialize
+        elif isinstance(value, list):
+            serialized_list = []
+            for item in value:
+                if isinstance(item, BaseModel):
+                    serialized_list.append(serialize_object(item))
+                else:
+                    serialized_list.append(item)
+            obj_dict[key] = serialized_list
+        else:
+            obj_dict[key] = value
+
+    dict_string = ",".join(f"{k}:{v}" for k, v in obj_dict.items())
+    return f"{class_name}:{{{dict_string}}}"
+
+
+def deserialize_object(data: str) -> BaseModel:
+    """
+    Deserializes a string in the format "<ClassName>:{key1:value1, key2:value2,...}"
+    back into an BaseModel object.  Handles UUIDs, datetimes, and
+    nested Pydantic models.
+
+    Args:
+        data: The string to deserialize.
+
+    Returns:
+        An R2RSerializable object.
+    """
+    class_name, dict_str = data.split(":", 1)
+    dict_str = dict_str.strip("{}")
+    pairs = dict_str.split(",") if dict_str else []
+
+    obj_dict: Dict[str, Any] = {}
+    for pair in pairs:
+        if not pair or ":" not in pair:
+            continue
+        k, v = pair.split(":", 1)
+        v = v.strip()  # important
+        obj_dict[k] = _convert_value(v)
+
+    logging_debug(f"Deserialized object: {class_name}")
+    if "[" in class_name:
+        class_name = class_name.split("[")[0]
+    logging_debug(f"trying object: {class_name}")
+    cls = globals().get(class_name)
+    if cls is None:
+        raise ValueError(f"Unknown class: {class_name}")
+
+    return cls(**obj_dict)
+
+
+def _convert_value(v: str) -> Any:
+    """
+    Helper function to convert a string to its appropriate Python type.
+    """
+    if v == "None":
+        return None
+    elif _is_uuid_string(v):
+        return uuid.UUID(v)
+    elif _is_datetime_string(v):
+        return datetime.fromisoformat(v)
+    elif _is_int_string(v):
+        return int(v)
+    elif (
+        v.startswith("{")
+        and v.endswith("}")
+        and ":" in v
+        and "{" not in v[1:-1]
+        and "}" not in v[1:-1]
+    ):  # nested object
+        # check if it is a json dict
+        try:
+            json.loads(v)
+            return json.loads(v)
+        except json.JSONDecodeError:
+            return deserialize_object(v)  # recursive call
+    elif v.startswith("[") and v.endswith("]"):  # list
+        v = v[1:-1]  # remove brackets
+        items = v.split(",") if v else []
+        deserialized_list = []
+        for item in items:
+            item = item.strip()
+            deserialized_list.append(_convert_value(item))  # recursive call
+        return deserialized_list
+    else:
+        return v  # Keep as string
+
+
+def _is_uuid_string(s: str) -> bool:
+    """Helper to check if a string is a valid UUID."""
+    try:
+        uuid.UUID(s)
+        return True
+    except ValueError:
+        return False
+
+
+def _is_datetime_string(s: str) -> bool:
+    """Helper to check if a string is a valid datetime in ISO format"""
+    try:
+        datetime.fromisoformat(s)
+        return True
+    except ValueError:
+        return False
+
+
+def _is_int_string(s: str) -> bool:
+    """Helper to check if a string is a valid int."""
+    try:
+        int(s)
+        return True
+    except ValueError:
+        return False
+
+
+def parse_url_query(url):
+    """
+    Parses a HTTP GET URL and returns (context_path, query_params_dict)
+
+    Args:
+        url: The URL string to parse (e.g., "http://example.com/path?foo=1&bar=2&foo=3")
+
+    Returns:
+        tuple: (context_path, query_params_dict)
+        - context_path: The path part of the URL (e.g., "/path")
+        - query_params_dict: Dictionary of query parameters where values are always lists
+                           (e.g., {"foo": ["1", "3"], "bar": ["2"]})
+    """
+    parsed = urlparse(url)
+    query_params = parse_qs(parsed.query)
+
+    # Convert values from lists to single values when there's only one value
+    # If you prefer to always keep values as lists, remove this comprehension
+    simplified_params = {k: v[0] if len(v) == 1 else v for k, v in query_params.items()}
+
+    return (parsed.path, simplified_params)

amqp/adapter/response_holder.py

@@ -1,15 +0,0 @@
-from dataclasses import dataclass
-from concurrent.futures import Future
-
-from aiohttp.web_response import Response
-from opentelemetry.trace import Span
-
-
-@dataclass
-class ResponseHolder:
-    response: Future[Response]
-    tracing_span: Span
-
-    def __init__(self, response: Future[Response], span: Span):
-        self.response = response
-        self.tracing_span = span

amqp/adapter/serializer.py

@@ -1,19 +1,23 @@
+"""
+Helper functions used to aid with (de)serialization of messages sent over AMQP.
+"""
+
 import struct
 from io import BytesIO
 from typing import List, Union, Dict
 
 
 def long_to_bytes(x: int) -> bytes:
-    return struct.pack('q', x)
+    return struct.pack("q", x)
 
 
 def bytes_to_long(b: bytes) -> int:
-    return struct.unpack('q', b)[0]
+    return struct.unpack("q", b)[0]
 
 
 def read_long(bis: BytesIO) -> int:
     try:
-        return struct.unpack('q', bis.read(8))[0]
+        return struct.unpack("q", bis.read(8))[0]
     except IOError:
         return 0
 
@@ -25,37 +29,37 @@ def object_or_list_as_string(value: Union[str, List[str]]) -> str:
 
 
 def map_as_string(map_: Dict[str, str]) -> str:
-    return ','.join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
+    return ",".join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
 
 
 def map_with_list_as_string(map_: Dict[str, List[str]]) -> str:
-    return ','.join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
+    return ",".join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
 
 
 def parse_map_string(input_str: str) -> Dict[str, str]:
     map_ = {}
-    if input_str.startswith('{') and input_str.endswith('}'):
-        for entry in input_str[1:-1].split(','):
+    if input_str.startswith("{") and input_str.endswith("}"):
+        for entry in input_str[1:-1].split(","):
             if len(entry) > 1:
-                key, value = entry.split('=')
+                key, value = entry.split("=")
                 map_[key] = value
     return map_
 
 
 def csv_as_list(input_str: str) -> List[str]:
-    return [item.strip() for item in input_str.strip('[]').split(',')]
+    return [item.strip() for item in input_str.strip("[]").split(",")]
 
 
 def add_to_map(map_: Dict[str, List[str]], token: str) -> None:
-    eq_idx = token.index('=')
+    eq_idx = token.index("=")
     if eq_idx > 0:
         key = token[:eq_idx]
-        map_[key] = csv_as_list(token[eq_idx+1:])
+        map_[key] = csv_as_list(token[eq_idx + 1 :])
 
 
 def parse_map_list_string(input_str: str) -> Dict[str, List[str]]:
     map_ = {}
-    if input_str.startswith('{') and input_str.endswith('}'):
-        for token in input_str[1:-1].split('],'):
+    if input_str.startswith("{") and input_str.endswith("}"):
+        for token in input_str[1:-1].split("],"):
             add_to_map(map_, token)
     return map_

amqp/adapter/service_message_factory.py

@@ -1,59 +1,92 @@
 import base64
 import logging
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.logging_utils import logging_error, logging_info
 from amqp.adapter.trace_info_adapter import TraceInfoAdapter
-from amqp.model.model import CleverMicroServiceMessage, ServiceMessageType
+from amqp.model.model import ServiceMessage
+from amqp.model.service_message_type import ServiceMessageType
 from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import parse_map_string, map_as_string
+from amqp.adapter.serializer import parse_map_string, map_as_string
 
 RS = "|"
 SPLIT_RS = r"\|"
 
 
-INVALID_MESSAGE = CleverMicroServiceMessage()  # default values mean invalid message
+INVALID_MESSAGE = ServiceMessage()  # default values mean invalid message
 
 
-class CleverMicroServiceMessageFactory:
+class ServiceMessageFactory:
+    """
+    build/serialize/deserialize CleverMicro Service Message class
+    """
 
     def __init__(self, name: str):
         self.process_name = name
 
-    def to_string(self, message: CleverMicroServiceMessage) -> str:
+    def to_string(self, message: ServiceMessage) -> str:
+        """
+        Convert ServiceMessage to human-readable format
+        :param message: ServiceMessage object
+        :return: as string
+        """
         if message is not None:
             return (
-                f"CleverMicroServiceMessage{{id={message.id.as_string()}|type={self.format_message_type(message)}|"
+                f"ServiceMessage{{id={message.id.as_string()}|type={self.format_message_type(message)}|"
                 f"recipient={message.recipient_name}|replyTo={message.reply_to}|"
                 f"traceInfo={message.trace_info}|body={base64.b64encode(message.message.encode()).decode() if message.payload_type & 0x80 else message.message}}}"
             )
         return "null"
 
-    def of(self, message_type: ServiceMessageType, payload: str, recipient_name: str) -> CleverMicroServiceMessage:
-        _id = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        return CleverMicroServiceMessage(
+    def of(
+        self, message_type: ServiceMessageType, payload: str, recipient_name: str
+    ) -> ServiceMessage:
+        """
+        Builds ServiceMessage object from relevant values.
+        :param message_type: enum Message type
+        :param payload: Message content, stored as-is
+        :param recipient_name: Recipient name (from the routing expression)
+        :return: ServiceMessage  object
+        """
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        return ServiceMessage(
             id=_id,
             payload_type=0,
             message_type=message_type,
             recipient_name=recipient_name,
             reply_to=self.process_name,
             trace_info=TraceInfoAdapter.create_produce_span(_id),
-            message=payload
+            message=payload,
         )
 
-    def as_base64(self, message_type: ServiceMessageType, payload: str, recipient_name: str) -> CleverMicroServiceMessage:
-        _id = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        return CleverMicroServiceMessage(
+    def as_base64(
+        self, message_type: ServiceMessageType, payload: str, recipient_name: str
+    ) -> ServiceMessage:
+        """
+        Builds ServiceMessage object from relevant values, stores content Base64 encoded.
+        :param message_type: enum Message type
+        :param payload: Message content, stored as Base64 string
+        :param recipient_name: Recipient name (from the routing expression)
+        :return: ServiceMessage  object
+        """
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        return ServiceMessage(
             id=_id,
             payload_type=0x80,
             message_type=message_type,
             recipient_name=recipient_name,
             reply_to=self.process_name,
             trace_info=TraceInfoAdapter.create_produce_span(_id.as_string()),
-            message=payload
+            message=payload,
         )
 
-    def from_bytes(self, input_bytes: bytes) -> CleverMicroServiceMessage:
-        print(f"CleverMicroServiceMessage >>> [{input_bytes.decode()}]")
+    def from_bytes(self, input_bytes: bytes) -> ServiceMessage:
+        """
+        Deserialize ServiceMessage from byte array serialized form.
+        :param input_bytes: serialized ServiceMessage
+        :return: ServiceMessage object represented by the input.
+        """
+        logging_info(f"ServiceMessage >>> [{input_bytes.decode()}]")
         input_str_array = input_bytes.decode().split(RS)
         try:
             i = 0
@@ -66,42 +99,58 @@ class CleverMicroServiceMessageFactory:
             i += 1
             reply_to = input_str_array[i] if i < len(input_str_array) else ""
             i += 1
-            trace_info = parse_map_string(input_str_array[i]) if i < len(input_str_array) else {}
+            trace_info = (
+                parse_map_string(input_str_array[i]) if i < len(input_str_array) else {}
+            )
             i += 1
             message = input_str_array[i] if i < len(input_str_array) else ""
-            return CleverMicroServiceMessage(
+            return ServiceMessage(
                 id=SnowflakeId.from_hex(_id),
                 payload_type=payload_type,
                 message_type=message_type,
                 recipient_name=recipient_name,
                 reply_to=reply_to,
                 trace_info=trace_info,
-                message=base64.b64decode(message).decode() if payload_type & 0x80 else message
+                message=(
+                    base64.b64decode(message).decode()
+                    if payload_type & 0x80
+                    else message
+                ),
             )
         except Exception as e:
-            logging.error(
+            logging_error(
                 "Unexpected ServiceMessage format. Expected 6 tokens separated by '|' in format: "
                 "[ID(64bit number)|MessageType(8bit number)|RecipientName(String)|"
                 "ReplyTo(String)|TraceInfo|Payload(String or Base64)], got: "
-                "[{}], reported problem: {}", input_bytes.decode(), str(e)
+                "[%s], reported problem: %s",
+                input_bytes.decode(),
+                str(e),
             )
             return INVALID_MESSAGE
 
-    def serialize(self, message: CleverMicroServiceMessage) -> bytes:
+    def serialize(self, message: ServiceMessage) -> bytes:
+        """
+        Serialize ServiceMessage to byte array.
+        :param message: ServiceMessage to serialize.
+        :return: serialized object.
+        """
         return (
             f"{message.id.as_string()}{RS}{self.format_message_type(message)}{RS}{message.recipient_name}{RS}"
             f"{message.reply_to}{RS}{map_as_string(message.trace_info)}{RS}"
             f"{base64.b64encode(message.message.encode()).decode() if message.payload_type != 0 else message.message}"
-        ).encode('utf-8')
+        ).encode("utf-8")
 
-
-    def format_message_type(self, message: CleverMicroServiceMessage) -> str:
+    def format_message_type(self, message: ServiceMessage) -> str:
         """
-        Formats the Enum message type and ecorates it with base64 flag at highest bit.
+        Formats the Enum message type and decorates it with base64 flag at highest bit.
         For compatibility with 0-based enum ordinals in Java subtract 1 from ordinal value.
         :param message: Service message to send
         :return: formatted message type
         """
-        num_type = (message.message_type.value if message.message_type else ServiceMessageType.UNKNOWN.value) - 1
-        fmt = ((num_type & 0x7F) | (message.payload_type & 0x80))
+        num_type = (
+            message.message_type.value
+            if message.message_type
+            else ServiceMessageType.UNKNOWN.value
+        ) - 1
+        fmt = (num_type & 0x7F) | (message.payload_type & 0x80)
         return f"{fmt:02X}"

amqp/adapter/test_amq_message_factory.py

@@ -1,68 +0,0 @@
-from datetime import datetime, timezone
-from unittest import TestCase
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-
-
-class AMQMessageFactoryTest(TestCase):
-    def test_from_bytes(self):
-        raw = "\002\027A64f3b5af4d00000a4000|m=POST|d=clever-pybook.localhost|p=/debug|h={Accept=[*/*],X-Forwarded-Host=[clever-pybook.localhost:8085],User-Agent=[curl/8.7.1],X-Forwarded-Proto=[http],X-Forwarded-For=[172.18.0.1],Host=[clever-pybook.localhost:8085],Accept-Encoding=[gzip],Content-Length=[1475],X-Forwarded-Port=[8085],X-Real-Ip=[172.18.0.1],X-Forwarded-Server=[c5945bef67d1],Content-Type=[multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v]}|t={traceparent=00-bed6eb7de0d8ccf4c80b145a4bfec357-8d1f3bafc471bf0f-01}|b=TEST_ME"
-        msg = AMQMessageFactory.from_bytes(raw.encode("utf-8"))
-        self.assertIsNotNone(msg.id)
-        self.assertEqual(msg.id.as_string(),'64F3B5AF4D00000A4000','SnowflakeID incorrectly parsed.')
-        self.assertEqual(msg.headers["Content-Type"][0], "multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v")
-
-    def test_generate_snowflake_id(self):
-        _id = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        self.assertEqual('00001000', _id.as_string()[-8:], 'Instance ID not set correctly')
-
-        _id = AMQMessageFactory(2).generate_snowflake_id()
-        self.assertEqual('00002000', _id.as_string()[-8:], 'Instance ID not set correctly')
-
-        _f = AMQMessageFactory(127)
-        _i0 = _f.generate_snowflake_id()
-        _i1 = _f.generate_snowflake_id()
-        _i2 = _f.generate_snowflake_id()
-        _i3 = _f.generate_snowflake_id()
-        self.assertEqual('0007F000', _i0.as_string()[-8:], 'Instance ID and sequence not set correctly')
-        self.assertEqual('0007F001', _i1.as_string()[-8:], 'Instance ID and sequence not set correctly')
-        self.assertEqual('0007F002', _i2.as_string()[-8:], 'Instance ID and sequence not set correctly')
-        self.assertEqual('0007F003', _i3.as_string()[-8:], 'Instance ID and sequence not set correctly')
-
-    def test_create_request_message(self):
-        _f = AMQMessageFactory(34)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        self.assertEqual("localhost", _req.domain, 'Domain not set correctly')
-        self.assertEqual("/test/me", _req.path, 'Context path not set correctly')
-        self.assertEqual("v1", _req.headers['k1'][0], "Headers not set correctly")
-        self.assertEqual(b'Quick brown fox', _req.body(), "Body not correctly decoded")
-
-    def test_serialize(self):
-        _f = AMQMessageFactory(34)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        _ser = _f.serialize(_req)
-        self.assertEqual(65, _ser[2])
-        _deser = _f.from_bytes(_ser)
-        self.assertEqual(_req.id.as_string(), _deser.id.as_string())
-        self.assertEqual("localhost", _deser.domain)
-        self.assertEqual("/test/me", _deser.path)
-        self.assertEqual(b"Quick brown fox", _deser.body())
-        self.assertEqual("v2", _deser.headers["k2"][0])
-
-    def test_amq_message_routing_key(self):
-        _f = AMQMessageFactory(35)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        _key = _f.amq_message_routing_key(_req)
-        self.assertEqual("localhost..test.me", _key, "Message routing key wrongly extracted from the message")
-
-    def test_get_timestamp_from_id(self):
-        _f = AMQMessageFactory(35)
-        _timestamp = _f.get_timestamp_from_id(_f.generate_snowflake_id())
-        _x_timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
-        self.assertLessEqual(_x_timestamp - _timestamp, 1, "Timestamp in SnowflakeId differs too much from current time")
-
-    def test_to_string(self):
-        _f = AMQMessageFactory(35)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        _as_str = _f.to_string(_req)
-        self.assertGreater(_as_str.index("domain='localhost'"), 0, "Expected content not found")

amqp/adapter/test_amq_route_factory.py

@@ -1,20 +0,0 @@
-from unittest import TestCase
-
-from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
-from amqp.model.model import AMQRoute
-
-
-class TestAMQRouteFactory(TestCase):
-    def test_from_string(self):
-        _f = AMQRouteFactory()
-        _r: AMQRoute = _f.from_string("amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0")
-        self.assertEqual("", _r.exchange)
-        self.assertEqual("amq-clever-pybook", _r.queue)
-        self.assertEqual(5, _r.timeout)
-        _r2: AMQRoute = _f.from_string("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#:5")
-        self.assertEqual(NULL_ROUTE, _r2)
-
-    def test_validate(self):
-        _f = AMQRouteFactory()
-        self.assertTrue(_f.validate("amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"))
-        self.assertFalse(_f.validate("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#"))

amqp/adapter/trace_info_adapter.py

@@ -1,14 +1,13 @@
 from dataclasses import dataclass
 from typing import Dict
 
+
 @dataclass
 class TraceInfoAdapter:
     """
-    This class should set up the Open Telemetry tracing stack. To be done later.
+    TODO This class should set up the Open Telemetry tracing stack. To be done later.
     """
 
     @staticmethod
     def create_produce_span(message_id: str) -> Dict[str, str]:
-        return {
-            "trace-id": message_id
-        }
+        return {"trace-id": message_id}

amqp/config/amq_configuration.py

@@ -1,10 +1,20 @@
 import configparser
+import inspect
 import os
 
+from amqp.adapter.logging_utils import logging_error, logging_warning
+
+"""
+Convert configuration from properties file to an object. 
+"""
+
 
 class AMQAdapter:
+    """
+    configuration with prefix 'cm.amq-adapter'
+    """
 
-    PREFIX: str = 'cm.amq-adapter.'
+    PREFIX: str = "cm.amq-adapter."
 
     def __init__(self, config: configparser.ConfigParser):
         """
@@ -14,23 +24,42 @@ class AMQAdapter:
 
         :param config: config parser to read configuration values
         """
-        self.generator_id = config.getint('CleverMicro-AMQ', self.PREFIX + 'generator-id', fallback=164)
-        self.service_name = config.get('CleverMicro-AMQ', self.PREFIX + 'service-name', fallback='amq-python-adapter-'+self.generator_id)
-        self.route_mapping = config.get('CleverMicro-AMQ', self.PREFIX + 'route-mapping', fallback=None)
+        self.generator_id = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "generator-id", fallback=164
+        )
+        self.service_name = config.get(
+            "CleverMicro-AMQ",
+            self.PREFIX + "service-name",
+            fallback="amq-python-adapter-" + str(self.generator_id),
+        )
+        self.route_mapping = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "route-mapping", fallback=None
+        )
+        self.require_authenticated_user = config.getboolean(
+            "CleverMicro-AMQ",
+            self.PREFIX + "require-authenticated-user",
+            fallback=False,
+        )
+        # these 2 shall be provided by Docker swarm, format is a random alphanumeric string
+        self.swarm_service_id = os.getenv(
+            "SWARM_SERVICE_ID", default="clever-amqp-service"
+        )
+        self.swarm_task_id = os.getenv("SWARM_TASK_ID", default="python-adapter")
 
     def adapter_prefix(self) -> str:
         return f"{self.service_name}-{self.generator_id}-"
 
 
-
 class Dispatch:
+    """
+    configuration with prefix 'cm.dispatch'
+    """
 
-    PREFIX: str = 'cm.dispatch.'
+    PREFIX: str = "cm.dispatch."
 
     def __init__(self, config: configparser.ConfigParser):
         """
         cm.dispatch.use-dlq=true
-        cm.dispatch.use-confirms=false
         cm.dispatch.amq-host=rabbitmq
         cm.dispatch.amq-port=5672
         cm.dispatch.amq-port-tls=5671
@@ -39,40 +68,103 @@ class Dispatch:
 
         :param config: config parser to read configuration values
         """
-        self.use_dlq = config.getboolean('CleverMicro-AMQ', self.PREFIX + 'use-dlq', fallback=False)
-        self.use_confirms = config.getboolean('CleverMicro-AMQ', self.PREFIX + 'use-confirms', fallback=False)
-        self.amq_host = config.get('CleverMicro-AMQ', self.PREFIX + 'amq-host', fallback='rabbitmq')
-        self.amq_port = config.getint('CleverMicro-AMQ', self.PREFIX + 'amq-port', fallback=5672)
-        self.amq_port_tls = config.getint('CleverMicro-AMQ', self.PREFIX + 'amq-port-tls', fallback=5671)
-        self.service_host = config.get('CleverMicro-AMQ', self.PREFIX + 'service-host', fallback='rabbitmq')
-        self.service_port = config.getint('CleverMicro-AMQ', self.PREFIX + 'service-port', fallback=8080)
-        self.rabbit_mq_user = os.environ["RABBIT_MQ_USER"]
-        self.rabbit_mq_password = os.environ["RABBIT_MQ_PASSWORD"]
+        self.use_dlq = config.getboolean(
+            "CleverMicro-AMQ", self.PREFIX + "use-dlq", fallback=False
+        )
+        self.amq_host = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "amq-host", fallback="rabbitmq"
+        )
+        self.amq_port = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "amq-port", fallback=5672
+        )
+        self.amq_port_tls = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "amq-port-tls", fallback=5671
+        )
+        self.service_host = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "service-host", fallback="localhost"
+        )
+        self.service_port = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "service-port", fallback=8080
+        )
+        self.rabbit_mq_user = os.getenv("RABBIT_MQ_USER", "springuser")
+        self.rabbit_mq_password = os.getenv("RABBIT_MQ_PASSWORD", "TheCleverWho")
+        self.rabbit_mq_url = (
+            f"amqp://{self.rabbit_mq_user}:{self.rabbit_mq_password}@{self.amq_host}/"
+        )
+        self.download_dir = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "download-dir", fallback="downloaded_files"
+        )
 
 
 class Backpressure:
+    """
+    configuration with prefix 'cm.backpressure'
+    """
 
-    PREFIX: str = 'cm.backpressure.'
+    PREFIX: str = "cm.backpressure."
 
     def __init__(self, config: configparser.ConfigParser):
         """
         cm.backpressure.threshold=20
         cm.backpressure.time-window=3000
-        cm.backpressure.management-service-url=management-service:8080
 
         :param config: config parser to read configuration values
         """
-        self.management_service_url = config.get('CleverMicro-AMQ', self.PREFIX + 'management-service-url', fallback='management-service:8080')
-        self.threshold = config.getint('CleverMicro-AMQ', self.PREFIX + 'threshold', fallback=20)
-        self.time_window = config.getint('CleverMicro-AMQ', self.PREFIX + 'time-window', fallback=3000)
+        self.threshold_threads = config.getint(
+            "CleverMicro-AMQ",
+            self.PREFIX + "threshold",
+            fallback=int(os.getenv("PARALLEL_WORKERS", 1)),
+        )
+        # time-window in seconds, duration between backpressure reports
+        self.time_window = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "time-window", fallback=10
+        )
+
+        # Define maximum CPU usage before triggering backpressure OVERLOAD alert
+        self.threshold_cpu = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "threshold-cpu", fallback=90
+        )
+        # Define the time window length, in seconds, during which, if CPU usage is consistently above the threshold, the system will trigger OVERLOAD alert
+        self.cpu_overload_duration = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "cpu-overload-duration", fallback=30
+        )
+        # Define the time window length, in seconds, during which, if no request is made to the Service, the system will trigger IDLE alert
+        self.idle_duration = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "idle-duration", fallback=30
+        )
 
 
 class AMQConfiguration:
+    """
+    Top level configuration context holder / object
+    """
 
-    def __init__(self):
+    def __init__(self, config_file_name: str = None):
         config = configparser.ConfigParser()
         # Read the application.properties file
-        config.read('config/application.properties')
+        if os.path.exists(config_file_name):
+            config.read(config_file_name)
+        else:
+            # Get the directory of the caller (the module where this function is called)
+            caller_frame = inspect.stack()[1]  # 0 is current frame, 1 is caller
+            caller_module = inspect.getmodule(caller_frame[0])
+            if caller_module:
+                caller_dir = os.path.dirname(os.path.abspath(caller_module.__file__))
+                config_file_path = os.path.join(caller_dir, config_file_name)
+                config.read(config_file_path)
+            else:
+                logging_error(f"Configuration file not found: {config_file_name}")
+                logging_warning(
+                    "Defaulting to preset values, which will likely be wrong, causing errors."
+                )
+
+        # Override values with environment variables
+        for section in config.sections():
+            for option in config.options(section):
+                # env_var = f"{section.upper()}_{option.upper()}"
+                if option in os.environ:
+                    config.set(section, option, os.environ[option])
+
         #
         # Add individual config areas
         #

amqp/config/application.properties

@@ -2,17 +2,30 @@
 # CleverMicro AMQ Adapter settings
 # ====================================================================
 [CleverMicro-AMQ]
-cm.amq-adapter.service-name=amq-adapter
-cm.amq-adapter.is-router=false
+cm.amq-adapter.service-name=amq-adapter-python
 cm.amq-adapter.generator-id=164
-cm.amq-adapter.route-mapping=
+cm.amq-adapter.route-mapping=amq-cleverswarm::amq-cleverswarm:cleverswarm.localhost.#:5:0
+cm.amq-adapter.require-authenticated-user=false
 
 cm.dispatch.use-dlq=true
 cm.dispatch.use-confirms=false
 cm.dispatch.amq-host=rabbitmq
 cm.dispatch.amq-port=5672
 cm.dispatch.amq-port-tls=5671
+cm.dispatch.download-dir=/tmp/downloaded_files
+# Not used for CleverSwarm nor CleverBRAG or in tight coupling mode.
+# applicable only in loose coupling mode, specify the destination where to forward the REST request
+#cm.dispatch.service-host=cleverswarm.localhost
+#cm.dispatch.service-port=8080
+#
+# Define maximum number of messages of being processed in parallel before triggering backpressure OVERLOAD alert
+cm.backpressure.threshold=5
+# Define maximum CPU usage before triggering backpressure OVERLOAD alert
+cm.backpressure.threshold-cpu=90
+# Define the time window between the Backpressure reports, in seconds
+cm.backpressure.time-window=10
+# Define the time window length, in seconds, during which, if CPU usage is consistently above the threshold, the system will trigger OVERLOAD alert
+cm.backpressure.cpu-overload-duration=30
+# Define the time window length, in seconds, during which, if no request is made to the Service, the system will trigger IDLE alert
+cm.backpressure.idle-duration=30
 
-cm.backpressure.threshold=20
-cm.backpressure.time-window=3000
-cm.backpressure.management-service-url=management-service:8080

amqp/config/application.properties.local

@@ -0,0 +1,19 @@
+# ====================================================================
+# CleverMicro AMQ Adapter settings
+# ====================================================================
+[CleverMicro-AMQ]
+cm.amq-adapter.service-name=amq-adapter-python
+cm.amq-adapter.generator-id=211
+cm.amq-adapter.route-mapping=amq-cleverswarm::amq-cleverswarm:localhost.#:5:0
+
+cm.dispatch.use-dlq=true
+cm.dispatch.use-confirms=false
+cm.dispatch.amq-host=localhost
+cm.dispatch.amq-port=5672
+cm.dispatch.amq-port-tls=5671
+# Not used for CleverSwarm nor CleverBRAG or in tight coupling mode.
+# applicable only in loose coupling mode, specify the destination where to forward the REST request
+#cm.dispatch.service-host=cleverswarm.localhost
+#cm.dispatch.service-port=8080
+
+cm.backpressure.threshold=20

amqp/example/Dockerfile

@@ -1,23 +0,0 @@
-FROM python:3.9-slim
-
-# Set the working directory in the container
-WORKDIR /app
-
-# Copy requirements.txt first to leverage Docker caching
-COPY requirements.txt ./
-
-# Install dependencies
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Copy the rest of the application code to the container
-COPY . .
-
-# Expose the port that Flask runs on
-EXPOSE 5000
-
-# Set environment variables for Flask
-ENV FLASK_APP=app.py
-ENV FLASK_RUN_HOST=0.0.0.0
-
-# Command to run the Flask application
-CMD ["flask", "run"]

amqp/example/clever_item.py

@@ -1,110 +0,0 @@
-from flask import Flask, request, jsonify
-import os
-import consul
-import configparser
-from werkzeug.utils import secure_filename,send_from_directory
-from opentelemetry import trace
-from opentelemetry.instrumentation.flask import FlaskInstrumentor
-from opentelemetry.instrumentation.requests import RequestsInstrumentor
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
-
-# Initialize Flask app
-app = Flask(__name__)
-
-# Configuration for file uploads
-UPLOAD_FOLDER = './amqp/example/uploaded_files'
-os.makedirs(UPLOAD_FOLDER, exist_ok=True)
-app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
-
-# Initialize Consul client
-consul_client = consul.Consul(host='192.168.229.128', port=8500)
-
-# OpenTelemetry configuration
-trace.set_tracer_provider(TracerProvider())
-otlp_exporter = OTLPSpanExporter(endpoint="http://192.168.229.128:4317", insecure=True)
-span_processor = BatchSpanProcessor(otlp_exporter)
-trace.get_tracer_provider().add_span_processor(span_processor)
-FlaskInstrumentor().instrument_app(app)
-RequestsInstrumentor().instrument()
-
-# In-memory store for CRUD operations
-data_store = {}
-
-def __init__(self):
-    config = configparser.ConfigParser()
-    # Read the application.properties file
-    config.read('application.properties')
-    #
-    # Add individual config areas
-    #
-    self.OTLP_endpoint = config.get('item.example.OTLP_endpoint')
-
-# CRUD Endpoints
-
-@app.route('/items', methods=['POST'])
-def create_item():
-    item_id = request.json.get('item_id')
-    data = request.json.get('data')
-
-    if not item_id or not data:
-        return jsonify({"error": "Item ID and data are required"}), 400
-
-    data_store[item_id] = data
-    return jsonify({"message": "Item created", "item_id": item_id})
-
-@app.route('/items', methods=['GET'])
-def get_all_items():
-    return jsonify({"items": data_store})
-
-@app.route('/items/<item_id>', methods=['GET'])
-def read_item(item_id):
-    data = data_store.get(item_id)
-    if data:
-        return jsonify({"item_id": item_id, "data": data})
-    return jsonify({"error": "Item not found"}), 404
-
-@app.route('/items/<item_id>', methods=['PUT'])
-def update_item(item_id):
-    data = request.json.get('data')
-    if not data:
-        return jsonify({"error": "Data is required"}), 400
-
-    if item_id in data_store:
-        data_store[item_id] = data
-        return jsonify({"message": "Item updated", "item_id": item_id})
-    return jsonify({"error": "Item not found"}), 404
-
-@app.route('/items/<item_id>', methods=['DELETE'])
-def delete_item(item_id):
-    if item_id in data_store:
-        del data_store[item_id]
-        return jsonify({"message": "Item deleted", "item_id": item_id})
-    return jsonify({"error": "Item not found"}), 404
-
-# File Upload and Download Endpoints
-
-@app.route('/upload', methods=['POST'])
-def upload_file():
-    if 'file' not in request.files:
-        return jsonify({"error": "No file part"}), 400
-
-    file = request.files['file']
-    if file.filename == '':
-        return jsonify({"error": "No selected file"}), 400
-
-    filename = secure_filename(file.filename)
-    file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
-    return jsonify({"message": "File uploaded", "filename": filename})
-
-@app.route('/download/<filename>', methods=['GET'])
-def download_file(filename):
-    file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
-    if os.path.exists(file_path):
-        return send_from_directory(app.config['UPLOAD_FOLDER'], filename, as_attachment=True)
-    return jsonify({"error": "File not found"}), 404
-
-
-if __name__ == '__main__':
-    app.run(debug=True)

amqp/healthcheck.py

@@ -0,0 +1,11 @@
+import sys
+import requests
+
+url = "http://localhost:8080/amq-adapter-healthcheck"
+headers = {"Authorization": "Bearer my_access_token"}
+params = {"limit": 10, "offset": 20}
+response = requests.get(url, headers=headers, params=params)
+if response.status_code == 200:
+    sys.exit(0)
+# Failed execution
+sys.exit(1)

amqp/model/model.py

@@ -1,67 +1,200 @@
 import base64
-
-from typing import List, Mapping, Dict
+import json
+from enum import Enum
+from typing import List, Dict
 from dataclasses import dataclass, field
-from enum import Enum, auto
 
+from amqp.model.service_message_type import ServiceMessageType
 from amqp.model.snowflake_id import SnowflakeId
 from amqp.router.utils import sanitize_as_rabbitmq_name
 
 RS = ":"
 
 
+"""
+Define the structure of messages used in Clever Micro
+"""
+
+
+class CleverMicroMessage:
+    DEFAULT_CONTENT_TYPE = ["application/octet-stream"]
+
+    def __init__(
+        self,
+        magic: str,
+        id: SnowflakeId,
+        m_field: str,
+        d_field: str,
+        p_field: str,
+        headers: Dict[str, List[str]],
+        trace_info: Dict[str, str],
+        base64body: bytes,
+    ):
+        self._magic = magic
+        self._id: SnowflakeId = id
+        self._m_field = m_field
+        self._d_field = d_field
+        self._p_field = p_field
+        self._headers: Dict[str, List[str]] = headers
+        self._trace_info: Dict[str, str] = trace_info
+        self._base64body: bytes = base64body
+
+    def magic(self):
+        return self._magic
+
+    def id(self):
+        return self._id
+
+    def m_field(self) -> str:
+        return self._m_field
+
+    def d_field(self) -> str:
+        return self._d_field
+
+    def p_field(self) -> str:
+        return self._p_field
+
+    def headers(self) -> Dict[str, List[str]]:
+        return self._headers
+
+    def trace_info(self) -> Dict[str, str]:
+        return self._trace_info
+
+    def base64body(self) -> bytes:
+        return self._base64body
+
+    def body(self) -> bytes:
+        return base64.b64decode(self.base64body())
+
+    def contentType(self) -> str:
+        return self.headers().get("Content-Type", self.DEFAULT_CONTENT_TYPE)[0]
+
+
+class DataMessage(CleverMicroMessage):
+    def __init__(
+        self,
+        magic: str,
+        id: SnowflakeId,
+        method: str,
+        domain: str,
+        path: str,
+        headers: Dict[str, List[str]],
+        trace_info: Dict[str, str],
+        base64body: bytes,
+    ):
+        super().__init__(
+            magic, id, method, domain, path, headers, trace_info, base64body
+        )
+
+    def method(self) -> str:
+        return self.m_field()
+
+    def domain(self) -> str:
+        return self.d_field()
+
+    def path(self) -> str:
+        return self.p_field()
+
+
+class DataResponse(CleverMicroMessage):
+    def __init__(
+        self,
+        magic: str,
+        id: SnowflakeId,
+        response_code: str,
+        error: str,
+        error_cause: str,
+        headers: Dict[str, List[str]],
+        trace_info: Dict[str, str],
+        base64body: bytes,
+    ):
+        super().__init__(
+            magic,
+            id,
+            response_code,
+            error,
+            error_cause,
+            headers,
+            trace_info,
+            base64body,
+        )
+
+    def response_code(self) -> str:
+        return self.m_field()
+
+    def error(self) -> str:
+        return self.d_field()
+
+    def error_cause(self) -> str:
+        return self.p_field()
+
+
 @dataclass(frozen=True)
 class AMQRoute:
     """
     Configuration item representing one AMQRoute. Adapter uses this to declare the Exchange
     and/or queue and bind the key as routing expression.
 
-    :param componentName: a service/component name that this route points to (owner of the route)
+    :param component_name: a service/component name that this route points to (owner of the route)
     :param exchange: RabbitMQ exchange name
     :param queue: RabbitMQ queue name
     :param key: RabbitMQ TOPIC Exchange routing key
     :param timeout: Timeout in milliseconds, how long to wait for reply, or not waiting if less then 1
-    :param validUntil: optional timestamp (millis from epoch) after which the route is ignored
+    :param valid_until: optional timestamp (millis from epoch) after which the route is ignored
     """
-    componentName: str
+
+    component_name: str
     exchange: str
     queue: str
     key: str
     timeout: int
-    validUntil: int
+    valid_until: int
 
     FORMAT = "Component-Name:Exchange-Name:Queue-Name:Routing-Key:Timeout:Valid-Until"
 
     def __str__(self):
         return (
-            f"{self.componentName}{RS}{self.exchange}{RS}{self.queue}{RS}{self.key}"
-            f"{RS}{self.timeout}{RS}{self.validUntil}"
+            f"{self.component_name}{RS}{self.exchange}{RS}{self.queue}{RS}{self.key}"
+            f"{RS}{self.timeout}{RS}{self.valid_until}"
         )
 
     def __eq__(self, other):
         if isinstance(other, AMQRoute):
-            return (self.componentName == other.componentName
-                    and self.queue == other.queue
-                    and self.key == other.key)
+            return (
+                self.component_name == other.component_name
+                and self.queue == other.queue
+                and self.key == other.key
+            )
         return False
 
     @property
     def primary_key(self):
-        return f"{self.componentName}{RS}{self.queue}{RS}{self.key}"
+        """
+        Build the unique identifier of the route
+        :return primary key
+        """
+        return f"{self.component_name}{RS}{self.queue}{RS}{self.key}"
 
     def as_string(self) -> str:
         return self.__str__()
 
 
 @dataclass
-class AMQMessage:
+class DataMessage:
+    """
+    The class represents a structure of a message that transfers the user request to the Service.
+    It may also be used to make a call/request between services.
+    The response to this call is in format of DataResponse class.
+    Please see DataMessageFactory for code to instantiate, serialize and deserialize the class
+    """
+
     magic: str
     id: SnowflakeId
     method: str
     domain: str
     path: str
-    headers: Mapping[str, List[str]]
-    trace_info: Mapping[str, str]
+    headers: dict[str, List[str]]
+    trace_info: dict[str, str]
     base64_body: bytes
 
     def routing_key(self) -> str:
@@ -71,30 +204,51 @@ class AMQMessage:
         return base64.b64decode(self.base64_body)
 
 
+class MultipartDataMessage(DataMessage):
+    def __init__(self, message: DataMessage, extra_data: dict):
+        super().__init__(
+            message.magic,
+            message.id,
+            message.method,
+            message.domain,
+            message.path,
+            message.headers,
+            message.trace_info,
+            message.base64_body,
+        )
+        self.extra_data = extra_data
+
+
 @dataclass
-class AMQResponse:
+class DataResponse:
+    """
+    The response message to RPC style call (note: the request is made with DataMessage)
+    Please see DataResponseFactory for code to instantiate, serialize and deserialize the class
+    """
+
     id: str
     response_code: int
     content_type: str
     response: bytes
-    error: str
-    error_cause: str
-    trace_info: Mapping[str, str]
+    error: str | None
+    error_cause: str | None
+    trace_info: dict[str, str]
 
     def body(self) -> bytes:
         return base64.b64decode(self.response)
 
 
-class ServiceMessageType(Enum):
-    ROUTING_DATA_REQUEST = auto()
-    ROUTING_DATA = auto()
-    ROUTING_DATA_REMOVE = auto()
-    ROUTING_DATA_REPLACE = auto()
-    UNKNOWN = auto()
+# Ensure backward compatibility
+AMQResponse = DataResponse
 
 
 @dataclass(frozen=True)
-class CleverMicroServiceMessage:
+class ServiceMessage:
+    """
+    The CleverMicro service message that is used to communicate the internal state of the AMQ channel.
+    In v0.2 it supports service discovery / route registration and Backpressure status notification.
+    """
+
     id: SnowflakeId = None
     payload_type: int = 0
     message_type: ServiceMessageType = ServiceMessageType.UNKNOWN
@@ -105,3 +259,49 @@ class CleverMicroServiceMessage:
 
     def is_valid(self) -> bool:
         return self.message_type is not None
+
+
+class ScalingRequestAlert(Enum):
+    OVERLOAD = "OVERLOAD"
+    IDLE = "IDLE"
+    UPDATE = "UPDATE"
+    SCALE_UP = "SCALE_UP"
+    SCSLE_DOWN = "SCALE_DOWN"
+
+
+@dataclass(frozen=True)
+class ScalingRequest:
+    service_id: str
+    task_id: str
+    max_availability: int
+    current_availability: int
+    request_type: ScalingRequestAlert
+    version: int = 1
+
+    def to_json(self) -> str:
+        """Converts the object to a JSON string matching the Java structure"""
+        return json.dumps(
+            {
+                "version": self.version,
+                "serviceId": self.service_id,
+                "taskId": self.task_id,
+                "maxAvailability": self.max_availability,
+                "currentAvailability": self.current_availability,
+                "requestType": self.request_type.value,  # Using .value for Enum serialization
+            },
+            indent=2,
+        )
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "ScalingRequest":
+        """Creates a ScalingRequest from a JSON string"""
+        data = json.loads(json_str)
+        return cls(
+            service_id=data["serviceId"],
+            task_id=data["taskId"],
+            max_availability=data["maxAvailability"],
+            current_availability=data["currentAvailability"],
+            request_type=ScalingRequestAlert(
+                data["requestType"]
+            ),  # Convert string to Enum
+        )

amqp/model/service_message_type.py

@@ -0,0 +1,10 @@
+from enum import Enum, auto
+
+
+class ServiceMessageType(Enum):
+    ROUTING_DATA_REQUEST = auto()
+    ROUTING_DATA = auto()
+    ROUTING_DATA_REMOVE = auto()
+    ROUTING_DATA_REPLACE = auto()
+    BACKPRESSURE = auto()
+    UNKNOWN = auto()

amqp/model/snowflake_id.py

@@ -2,25 +2,31 @@ import struct
 
 
 class SnowflakeId:
-    SNOWFLAKE_ID_WIDTH = 10  # with in bytes (8 bits each), can be more than long int width (8 bytes)
+    """
+    see https://en.wikipedia.org/wiki/Snowflake_ID
+    """
 
-    def __init__(self, _input: bytes = None):
+    SNOWFLAKE_ID_WIDTH = (
+        10  # with in bytes (8 bits each), can be more than long int width (8 bytes)
+    )
+
+    def __init__(self, hi_bits_or_bytes=None, lo_bits=None):
         self.bits = [0, 0]
-        if _input is not None:
-            i = 0
-            shift = 8  # starting from Hi value, which is only 2 bytes wide
-            n = 1
-            while i < len(_input) and i <= self.SNOWFLAKE_ID_WIDTH:
-                self.bits[n] |= _input[i] << shift
-                if shift == 0:
-                    n -= 1
-                    shift = 56
-                else:
-                    shift -= 8
-                i += 1
-
-    def __init__(self, hi_bits: int, lo_bits: int):
-        self.bits = [lo_bits, hi_bits]
+        if lo_bits is None:
+            if hi_bits_or_bytes is not None:
+                i = 0
+                shift = 8  # starting from Hi value, which is only 2 bytes wide
+                n = 1
+                while i < len(hi_bits_or_bytes) and i <= self.SNOWFLAKE_ID_WIDTH:
+                    self.bits[n] |= hi_bits_or_bytes[i] << shift
+                    if shift == 0:
+                        n -= 1
+                        shift = 56
+                    else:
+                        shift -= 8
+                    i += 1
+        else:
+            self.bits = [lo_bits, hi_bits_or_bytes]
 
     @staticmethod
     def from_hex(_input):
@@ -36,13 +42,18 @@ class SnowflakeId:
         for i in range(0, expected_length, 2):
             if i == expected_length - (2 * 8):
                 n = 0
-            bits[n] = (bits[n] << 8) | int(_input[i:i + 2], 16)
+            bits[n] = (bits[n] << 8) | int(_input[i : i + 2], 16)
 
         return SnowflakeId(bits[1], bits[0])
 
     def __str__(self):
         _hex = f"{self.bits[1]:016X}{self.bits[0]:016X}"
-        return _hex if len(_hex) <= 2 * SnowflakeId.SNOWFLAKE_ID_WIDTH else _hex[(-2 * SnowflakeId.SNOWFLAKE_ID_WIDTH):]
+        _val = (
+            _hex
+            if len(_hex) <= 2 * SnowflakeId.SNOWFLAKE_ID_WIDTH
+            else _hex[(-2 * SnowflakeId.SNOWFLAKE_ID_WIDTH) :]
+        )
+        return _val.lower()
 
     def __eq__(self, other):
         if isinstance(other, SnowflakeId):
@@ -59,7 +70,7 @@ class SnowflakeId:
     def get_bytes(self):
         high = struct.pack("!Q", self.bits[1])
         low = struct.pack("!Q", self.bits[0])
-        return high[2:] + low
+        return high[-2:] + low
 
     def get_bits(self):
         return self.bits

amqp/rabbitmq/rabbit_mq_client.py

@@ -0,0 +1,237 @@
+import logging
+
+import aio_pika
+from aio_pika.abc import AbstractRobustQueue, AbstractRobustExchange, ExchangeType
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.logging_utils import (
+    logging_error,
+    logging_debug,
+    logging_info,
+    logging_warning,
+)
+from amqp.model.model import DataMessage, AMQRoute
+from amqp.router.router_base import (
+    AMQ_REPLY_TO_EXCHANGE,
+    DLQ_EXCHANGE,
+    AMQ_RPC_EXCHANGE,
+)
+from amqp.router.router_consumer import RouterConsumer
+from amqp.router.utils import sanitize_as_rabbitmq_name
+
+
+class RabbitMQClient:
+    PREFETCH_COUNT = 1
+    DLQ_ARGS = {"x-dead-letter-exchange": DLQ_EXCHANGE}
+
+    def __init__(self, configuration):
+        self.channel: aio_pika.RobustChannel | None = None
+        self.connection: aio_pika.RobustConnection | None = None
+        self.rpc_exchange: aio_pika.RobustExchange | None = None
+        self.reply_exchange: aio_pika.RobustExchange | None = None
+        self.amq_configuration = configuration
+        logging.info("*******************************************")
+        logging.info(
+            "******** RabbitMQClient.init(): %s",
+            self.amq_configuration.dispatch.amq_host,
+        )
+        logging.info("*******************************************")
+        self.router = RouterConsumer(configuration)
+        # this.meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
+
+        self.reply_to_exchange: AbstractRobustExchange | None = (
+            None  # when sending response to RPC request
+        )
+        self.reply_to_queue: AbstractRobustQueue | None = (
+            None  # when receiving response to RPC request
+        )
+
+    async def init(self, loop) -> AbstractRobustExchange:
+        await self.setup_amq_channel(loop)
+        await self.router.init_routing_paths_consumer(self.router.channel)
+        cm_reply_to_queue = self.router.get_reply_to_queue_name()
+        self.reply_to_queue = await self.channel.declare_queue(
+            name=cm_reply_to_queue, durable=True, exclusive=False, auto_delete=False
+        )
+        self.reply_to_exchange = await self.channel.declare_exchange(
+            AMQ_REPLY_TO_EXCHANGE, type=ExchangeType.DIRECT
+        )
+        await self.reply_to_queue.bind(
+            exchange=self.reply_to_exchange, routing_key=cm_reply_to_queue
+        )
+        return self.reply_to_exchange
+
+    def cleanup(self):
+        logging_info("RabbitMQClient.cleanup()")
+        self.router.cleanup()  # de-register route(s)
+        if self.router.is_open(self.channel):
+            try:
+                self.channel.close()
+                if not self.connection.is_closed:
+                    self.connection.close()
+            except Exception as e:
+                logging_error(
+                    "RabbitMQClient.PreDestroy cleanup - connection close error: %s",
+                    e,
+                )
+
+    async def register_inbound_routes(self, route_mapping: str, callback):
+        """
+        Register inbound routes for the AMQ adapter. The route mapping is a comma-separated string that contains
+        list of routes. A route is the mapping of domain and path to the exchange and queue name.
+        This method declare (and thus create) the queue and exchange, bind them together using the
+        regex-like routing expression given in the route.
+        After this the Service is listening on (consuming from) the queue for incoming messages.
+        """
+        if self.router.is_open(self.channel):
+            queue_args = (
+                self.DLQ_ARGS if self.amq_configuration.dispatch.use_dlq else None
+            )
+            logging_info(
+                "RabbitMQClient register routes, cfg mapping: [%s]", route_mapping
+            )
+            # convert the comma-separated string into list of AMQRoute objects
+            requested_routes = self.router.unwrap_route_list(route_mapping)
+            for route in requested_routes:
+                try:
+                    # Note: AMQRoute allows one of the exchange or queue to be None, in which case the default is used
+                    queue_name = (
+                        route.queue
+                        if route.queue
+                        else self.router.get_consume_queue_name()
+                    )
+                    logging_info("RabbitMQClient register route: [%s]", route)
+                    queue: AbstractRobustQueue = await self.channel.declare_queue(
+                        name=queue_name,
+                        durable=True,
+                        exclusive=False,
+                        auto_delete=False,
+                        arguments=queue_args,
+                    )
+                    _exchange_name: str = (
+                        route.exchange if route.exchange else AMQ_RPC_EXCHANGE
+                    )
+                    exchange: AbstractRobustExchange = (
+                        await self.channel.declare_exchange(
+                            name=_exchange_name, type=ExchangeType.TOPIC
+                        )
+                    )
+                    await queue.bind(exchange, route.key)
+                    _c_tag = await queue.consume(callback, no_ack=False)
+                    # Register the route with the router and publish its detail to other adapters
+                    await self.router.register_route(
+                        AMQRoute(
+                            route.component_name,
+                            _exchange_name,
+                            queue_name,
+                            sanitize_as_rabbitmq_name(route.key),
+                            route.timeout,
+                            route.valid_until,
+                        )
+                    )
+                    logging_info(
+                        " [%s] AMQ connection initialized for route: %s, listens on: %s/%s",
+                        _c_tag,
+                        route,
+                        exchange,
+                        queue_name,
+                    )
+                except Exception as err:
+                    logging_error("Callback registration INCOMPLETE, %s", err)
+        else:
+            logging_warning("Channel is null, cannot register routes.")
+
+    async def register_data_reply_callback(self, rpc_callback):
+        _c_tag = await self.reply_to_queue.consume(callback=rpc_callback, no_ack=False)
+        logging_info(
+            " [%s] RabbitMQClient register data reply callback on queue: %s",
+            _c_tag,
+            self.reply_to_queue.name,
+        )
+
+    async def setup_amq_channel(self, loop):
+        """
+        Set up the RabbitMQ connection
+        :return: nothing, it applies changes to instance variables
+        """
+        self.connection = await aio_pika.connect_robust(
+            host=self.amq_configuration.dispatch.amq_host,
+            port=self.amq_configuration.dispatch.amq_port,
+            login=self.amq_configuration.dispatch.rabbit_mq_user,
+            password=self.amq_configuration.dispatch.rabbit_mq_password,
+            loop=loop,
+        )
+        logging_debug(
+            "RabbitMQClient.setupAMQ, got connection, open=%s",
+            self.connection.is_closed,
+        )
+        self.channel = await self.connection.channel()
+        await self.channel.set_qos(prefetch_count=self.PREFETCH_COUNT)
+        await self.router.init_internal_routing_paths(self.channel)
+        self.router.set_route_added_notifier(
+            lambda route: logging_debug(
+                " RouteAdded, setting up metrics: amqp.adapter.exchange.%s",
+                route.exchange,
+            )
+        )
+
+    async def send_message(self, message: DataMessage):
+        """
+        The API routine that sends a datagram message (DataMessage) to the destination. The Destination
+        is determined from the DataMessage metadata section, using `domain` and `path` as key.
+        :param message: message to send
+        :return: nothing (void) as submitting message to RabbitMQ exchange does not return anything
+         meaningful. There will be asynch ACK later that will confirm the message was sent to RMQ.
+        """
+        try:
+            route = self.router.find_route_by_message(message)
+            if route and self.rpc_exchange:
+                if route.timeout <= 0:
+                    # No response expected
+                    pika_message: aio_pika.message.Message = aio_pika.message.Message(
+                        body=DataMessageFactory.serialize(message),
+                        content_encoding="utf-8",
+                        delivery_mode=2,
+                    )
+                    await self.rpc_exchange.publish(
+                        message=pika_message,
+                        routing_key=self.router.get_routing_key(
+                            message
+                        ),  # context path is a routing key
+                    )
+                    logging_info(
+                        f"Msg Publish (no-reply), messageId: {message.id}, to: {route}"
+                    )
+                else:
+                    # This is RPC call, there should be response
+                    pika_message: aio_pika.message.Message = aio_pika.message.Message(
+                        body=DataMessageFactory.serialize(message),
+                        content_encoding="utf-8",
+                        delivery_mode=2,
+                        correlation_id=str(message.id),
+                        reply_to=self.router.get_reply_to_queue_name(),
+                    )
+                    await self.rpc_exchange.publish(
+                        message=pika_message,
+                        routing_key=self.router.get_routing_key(
+                            message
+                        ),  # context path is a routing key
+                    )
+                    logging_info(
+                        "Msg Publish (RPC call), messageId: %s, to: %s",
+                        message.id.as_string(),
+                        route.as_string(),
+                    )
+            else:
+                logging_warning(
+                    "Message not sent, no route for %s/%s", message.domain, message.path
+                )
+
+        except Exception as err:
+            logging_error("Send message failed: %s", err)
+
+    async def request_routes(self):
+        await self.router.request_routes_from_adapters()
+
+    def find_route(self, domain: str, path: str) -> AMQRoute:
+        return self.router.find_route(domain, path)

amqp/rabbitmq/rabbit_mq_consumer.py

@@ -1,143 +0,0 @@
-import logging
-import pika
-from pika.exchange_type import ExchangeType
-
-from amqp.model.model import AMQRoute
-from amqp.rabbitmq.rabbit_mq_producer import RabbitMQProducer
-from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE, DLQ_EXCHANGE, AMQ_RPC_EXCHANGE
-from amqp.router.router_consumer import RouterConsumer
-from amqp.router.utils import sanitize_as_rabbitmq_name
-
-
-class RabbitMQConsumer(RabbitMQProducer):
-    PREFETCH_COUNT = 1
-    DLQ_ARGS = {"x-dead-letter-exchange": DLQ_EXCHANGE}
-
-    def __init__(self, configuration):
-        super().__init__(configuration, RouterConsumer(configuration))
-
-    def cleanup(self):
-        logging.info(" [*] RabbitMQConsumer.cleanup()")
-        self.router.cleanup()  # de-register route(s)
-        if self.router.is_open(self.channel):
-            try:
-                self.channel.close()
-            except (pika.connection.exceptions.AMQPError, pika.connection.exceptions.ChannelClosedByBroker) as e:
-                logging.error(" [E] RabbitMQConsumer.PreDestroy cleanup - connection close error: %s", e)
-
-    def register_routes(self):
-        if self.router.is_open(self.channel):
-            queue_args = self.DLQ_ARGS if self.amq_configuration.dispatch.use_dlq else None
-            route_mapping = self.amq_configuration.amq_adapter.route_mapping
-            logging.info(" [*] RabbitMQConsumer register routes, cfg mapping: [%s]", route_mapping)
-            requested_routes = self.router.unwrap_route_list(route_mapping)
-            for route in requested_routes:
-                queue_name = route.queue() if route.queue() else self.router.get_consume_queue_name()
-                logging.info(" [*] RabbitMQConsumer register route: [%s]", route)
-                self.channel.queue_declare(queue_name, durable=True, exclusive=False,
-                                           auto_delete=False, arguments=queue_args)
-                exchange = route.exchange() if route.exchange() else AMQ_RPC_EXCHANGE
-                self.channel.exchange_declare(exchange, exchange_type=pika.exchange_type.ExchangeType.topic)
-                self.channel.queue_bind(queue_name, exchange, route.key())
-                self.router.register_route(
-                    AMQRoute(
-                        route.component_name(), exchange, queue_name,
-                        sanitize_as_rabbitmq_name(route.key()),
-                        route.timeout(), route.valid_until()
-                    )
-                )
-                logging.info(" [*] AMQ connection initialized for route: %s, listens on: %s/%s",
-                             route, exchange, queue_name)
-        else:
-            logging.warning(" [W] Channel is null, cannot register routes.")
-
-    def register_deliver_callback(self, callback_provider):
-        try:
-            for route in self.router.get_own_routes():
-                self.channel.basic_consume(
-                    queue=route.queue(),
-                    on_message_callback=callback_provider(self.channel),
-                    auto_ack=False
-                )
-                logging.info(" [*] Consumer Waiting for messages on: %s", route.queue())
-        except pika.connection.exceptions.AMQPError as err:
-            logging.error(" [E] Callback registration INCOMPLETE, %s", err)
-
-    def register_rpc_handler(self, rpc_consumer):
-        cm_reply_to_queue = self.router.get_reply_to_queue_name()
-        self.channel.queue_declare(cm_reply_to_queue, durable=True, exclusive=False, auto_delete=False)
-        self.channel.exchange_declare(
-            AMQ_REPLY_TO_EXCHANGE, exchange_type=pika.exchange_type.ExchangeType.direct
-        )
-        self.channel.queue_bind(cm_reply_to_queue, AMQ_REPLY_TO_EXCHANGE, cm_reply_to_queue)
-        self.channel.basic_consume(
-            queue=cm_reply_to_queue,
-            on_message_callback=rpc_consumer,
-            auto_ack=False
-        )
-
-    """
-    def limit_rate_inside_time_window(self, message: AMQMessage) -> int:
-        next_batch_size = BackpressureSubscriber.NO_OP
-        now = time.time_ns() // 1000000
-        queue_size = self.getQueueSize(message)
-
-        # 1. Backpressure? if unprocessed count exceeds threshold, wait for some time
-        if queue_size > THRESHOLD:
-            # TODO - signal queue backpressure explicitly to upstream/metrics
-            # wait: estimated-per-message-processing-time * SQRT(over-the-limit)
-            self.wait_for_milliseconds(
-                (WINDOW / self.request_for_window_duration) * (queue_size - THRESHOLD) ** 0.5)
-            queue_size = self.getQueueSize(message)  # refresh queue size after wait
-
-        # 2. batch complete? determine next batch size and wait till window end. reset window.
-        if self.fulfilled >= self.request_for_window_duration:
-            next_batch_size = self.calculateNextBatchSize(queue_size)
-            self.request_for_window_duration = next_batch_size
-            remaining_time = (self.window_started_at + WINDOW) - now
-            if remaining_time > 0:
-                self.wait_for_milliseconds(remaining_time)
-                queue_size = self.getQueueSize(message)  # refresh queue size after wait
-            self.window_started_at = time.time_ns() // 1000000
-            self.fulfilled = 0
-
-        # 3. Window elapsed? Reset window & recalculate batch size
-        if now >= self.window_started_at + WINDOW:
-            if next_batch_size == BackpressureSubscriber.NO_OP:
-                next_batch_size = self.calculateNextBatchSize(queue_size)
-                self.request_for_window_duration = next_batch_size
-            self.window_started_at = time.time_ns() // 1000000
-            self.fulfilled = 0
-
-        # 4. send the message from the current batch
-        self.send_message(message)
-        self.fulfilled += 1
-
-        # 5. report and setup state for the next call
-        end = time.time_ns() // 1000000
-        self.on_next_timestamp = end
-        return next_batch_size
-
-    def calculateNextBatchSize(self, queue_size: int) -> int:
-        next_batch_size = self.request_for_window_duration
-        if queue_size <= THRESHOLD:
-            # processing proceeds well, can increase the rate
-            next_batch_size += self.subscriber.increaseRate(self.request_for_window_duration)
-            logging.debug(f"INCREASE rate to {next_batch_size}. qSize={queue_size}")
-        else:
-            decrease = self.subscriber.backoff(queue_size - THRESHOLD)
-            if self.request_for_window_duration > decrease:
-                next_batch_size -= decrease
-            logging.debug(f"DECREASE rate to {next_batch_size}. qSize={queue_size}")
-        return next_batch_size
-    """
-    """
-    def getQueueSize(self, message: AMQMessage) -> int:
-        try:
-            route_queue_name = self.router.findRoute(message)
-            queue_name = route_queue_name.queue if route_queue_name else self.TASK_DISPATCH_QUEUE_NAME
-            logger.info(f"Get Queue size for {queue_name}")
-            return max(len(self.awaitingACK), self.channel.message_count(queue_name))
-        except Exception:
-            return len(self.awaitingACK)
-    """

amqp/rabbitmq/rabbit_mq_producer.py

@@ -1,134 +0,0 @@
-import logging
-import time
-
-import pika
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import AMQMessage, AMQRoute
-from amqp.router.router_producer import RouterProducer
-
-
-class RabbitMQProducer:
-    PREFETCH_COUNT = 1
-    THRESHOLD = 1000
-    WINDOW = 5000  # 5 seconds
-
-    def __init__(self, amq_configuration: AMQConfiguration, router: RouterProducer = None):
-        self.channel = None
-        self.connection = None
-        self.request_for_window_duration = None
-        self.amq_configuration = amq_configuration
-        logging.info("*******************************************")
-        logging.info("******** RabbitMQProducer.init() ********** %s - %s - %s",
-                     self.amq_configuration.dispatch.amq_host,
-                     self.amq_configuration.dispatch.use_confirms,
-                     self.WINDOW)
-        logging.info("*******************************************")
-        self.router = router or RouterProducer(amq_configuration)
-
-        # this.meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
-
-        self.setup_amq_channel()
-        self.router.init_routing_paths(self.channel)
-
-        #  self.backpressure_per_exchange = defaultdict(lambda: Gauge(
-        #    f"amqp.adapter.exchange.{exchange}", registry=CollectorRegistry()))
-
-        self.router.set_route_added_notifier(
-            lambda route: logging.info(" [DBG] ---------- RouteAdded, setting up metrics: amqp.adapter.exchange.%s",
-                                       route.exchange))
-
-        self.request_for_window_duration = self.THRESHOLD // 2
-        self.window_started_at = time.time()
-        self.on_next_timestamp = self.window_started_at
-        self.fulfilled = 0
-
-    def cleanup(self):
-        if self.connection:
-            try:
-                self.connection.close()
-            except Exception as e:
-                logging.error("RabbitMQConsumer.PreDestroy cleanup - connection close error: %s", e)
-
-    def setup_amq_channel(self):
-        """
-        Set up the RabbitMQ connection
-        :return: nothing, it applies changes to instance variables
-        """
-        factory = pika.ConnectionParameters(
-            host=self.amq_configuration.dispatch.amq_host,
-            username=self.amq_configuration.dispatch.rabbit_mq_user,
-            password=self.amq_configuration.dispatch.rabbit_mq_password
-        )
-        self.connection = pika.BlockingConnection(factory)
-        logging.info("RabbitMQProducer.setupAMQ, got connection, open=%s", self.connection.is_open)
-        self.connection.add_on_connection_blocked_callback(self.blocked_listener)
-        self.connection.add_on_connection_unblocked_callback(self.unblocked_listener)
-        self.channel = self.connection.channel()
-        self.channel.basic_qos(prefetch_count=self.PREFETCH_COUNT)
-        if self.amq_configuration.dispatch.use_confirms():
-            self.setup_local_ack()
-        self.channel.add_on_return_callback(self.return_listener)
-
-    def send_message(self, message: AMQMessage):
-        """
-        The API routine that sends a datagram message (AMQMessage) to the destination. The Destination
-        is determined from the AMQMessage metadata section, using `domain` and `path` as key.
-        :param message: message to send
-        :return: nothing (void) as submitting message to RabbitMQ exchange does not return anything
-         meaningful. There will be asynch ACK later that will confirm the message was sent to RMQ.
-        """
-        try:
-            route = self.router.find_route_by_message(message)
-            if route:
-                if route.timeout < 0:
-                    # No response expected
-                    self.channel.basic_publish(
-                        route.exchange,
-                        self.router.get_routing_key(message),  # context path is a routing key
-                        properties=pika.BasicProperties(delivery_mode=2),
-                        body=AMQMessageFactory.serialize(message)
-                    )
-                    logging.info(f" [x] basicPublish (no-reply), messageId: {message.id}, to: {route}")
-                else:
-                    # This is RPC call, there should be response
-                    props = pika.BasicProperties(
-                        correlation_id=message.id,
-                        reply_to=self.router.get_reply_to_queue_name()
-                    )
-                    self.channel.basic_publish(
-                        route.exchange,
-                        self.router.get_routing_key(message),  # context path is a routing key
-                        properties=props,
-                        body=AMQMessageFactory.serialize(message)
-                    )
-                    logging.info(" [x] basicPublish (RPC call), messageId: %s, to: %s",
-                                 message.id.as_string(), route.as_string())
-            else:
-                logging.warning("Message not sent, no route for {}/{}", message.domain, message.path)
-
-        except Exception as err:
-            logging.error("Send message failed: %s", err)
-
-    def request_routes(self):
-        self.router.request_routes_from_adapters()
-
-    def setup_local_ack(self):
-        self.channel.confirm_delivery()
-
-    def find_route(self, domain: str, path: str) -> AMQRoute:
-        return self.router.find_route(domain, path)
-
-    def return_listener(self, return_message):
-        logging.error(
-            f" [E] ****** (NOT DELIVERED) ID={return_message.properties.message_id}, "
-            f"Exchg=[{return_message.exchange}], Key=[{return_message.routing_key}], "
-            f"replyTxt={return_message.reply_text}, length={len(return_message.body) if return_message.body else 0}"
-        )
-
-    def blocked_listener(self, reason: str):
-        logging.error(f" [E] CHANNEL BLOCKED, reason={reason}")
-
-    def unblocked_listener(self):
-        logging.info(" [*] CHANNEL UNBLOCKED")

amqp/router/route_database.py

@@ -2,6 +2,7 @@ import logging
 import re
 from typing import Set
 
+from amqp.adapter.logging_utils import logging_info
 from amqp.model.model import AMQRoute
 from amqp.router.utils import sanitize_as_rabbitmq_name
 
@@ -14,26 +15,46 @@ class RouteDatabase:
         return self.routes
 
     def pattern(self, routing_key: str) -> re.Pattern:
-        regex = r"\.".join(
-            [
-                r".*" if token == "" else r"#" if token == "#" else token
-                for token in routing_key.split(".")
-            ]
-        )
-        if routing_key.endswith("."):
-            regex += r"\."
+        breaker = False
+        tokens = routing_key.split(".")
+        counter = len(tokens)
+        regex_parts = []
+
+        for token in tokens:
+            counter -= 1
+            if not breaker:
+                if token == "":
+                    res = "[^\\.]*"
+                elif token == "#":
+                    breaker = True
+                    res = ".*$"
+                else:
+                    res = token
+                if counter > 0 and not breaker:
+                    res += "\\."
+                regex_parts.append(res)
+            else:
+                regex_parts.append("")
+
+        regex = "".join(filter(lambda x: x != "", regex_parts))
+        if not routing_key:
+            return re.compile(".*")
         return re.compile(regex)
 
-    def matches(self, routing_key_from_route: str, routing_key_from_message: str) -> bool:
+    def matches(
+        self, routing_key_from_route: str, routing_key_from_message: str
+    ) -> bool:
         route_pattern = self.pattern(routing_key_from_route)
-        return bool(route_pattern.match(sanitize_as_rabbitmq_name(routing_key_from_message)))
+        return bool(
+            route_pattern.match(sanitize_as_rabbitmq_name(routing_key_from_message))
+        )
 
     def wrap_routes(self) -> str:
         return ",".join(str(route) for route in self.routes)
 
     def find_route(self, domain: str, path: str) -> AMQRoute | None:
         routing_key = sanitize_as_rabbitmq_name(f"{domain}.{path}")
-        logging.info(f" [DBG] findRoute key:{routing_key} in:{self.wrap_routes()}")
+        logging_info(f"findRoute key:{routing_key} in:{self.wrap_routes()}")
         for route in self.routes:
             if self.matches(route.key, routing_key):
                 return route

amqp/router/router_base.py

@@ -2,7 +2,8 @@ import logging
 from typing import Callable, List, Set
 
 from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
-from amqp.model.model import AMQRoute, AMQMessage
+from amqp.adapter.logging_utils import logging_error, logging_debug, logging_info
+from amqp.model.model import AMQRoute, DataMessage
 from amqp.router.route_database import RouteDatabase
 from amqp.router.utils import sanitize_as_rabbitmq_name
 
@@ -26,7 +27,7 @@ class RouterBase:
         self.route_added_notifier: Callable[[AMQRoute], None] | None = None
         self.route_removed_notifier: Callable[[AMQRoute], None] | None = None
 
-    def get_routing_key(self, message: AMQMessage) -> str:
+    def get_routing_key(self, message: DataMessage) -> str:
         return sanitize_as_rabbitmq_name(f"{message.domain}.{message.path}")
 
     def wrap_routes(self) -> str:
@@ -36,8 +37,11 @@ class RouterBase:
         return ",".join(str(route) for route in self.own_routes)
 
     def unwrap_route_list(self, route_string: str) -> List[AMQRoute]:
-        logging.info(" [DBG] RouterMaster.unwrapRoutes, route(s): %s", route_string)
-        routes = [AMQRouteFactory.from_string(route_str) for route_str in route_string.split(",")]
+        logging_debug("RouterMaster.unwrapRoutes, route(s): %s", route_string)
+        routes = [
+            AMQRouteFactory.from_string(route_str)
+            for route_str in route_string.split(",")
+        ]
         return [route for route in routes if route != NULL_ROUTE]
 
     def get_routes(self) -> Set[AMQRoute]:
@@ -46,37 +50,48 @@ class RouterBase:
     def find_route(self, domain: str, path: str) -> AMQRoute | None:
         return self.route_database.find_route(domain, path)
 
-    def find_route_by_message(self, message: AMQMessage) -> AMQRoute | None:
+    def find_route_by_message(self, message: DataMessage) -> AMQRoute | None:
         return self.route_database.find_route(message.domain, message.path)
 
     def add_routes(self, message: str) -> None:
         try:
             amq_route_list = self.unwrap_route_list(message)
-            logging.info(" [DBG] RouterMaster.addRoute [%d]: %s", len(amq_route_list), message)
+            logging_info(
+                " [DBG] RouterMaster.addRoute [%d]: %s", len(amq_route_list), message
+            )
             for route in amq_route_list:
-                logging.info(" [*] RouterMaster.addRoute, adding route: %s", route)
+                logging_info("RouterMaster.addRoute, adding route: %s", route)
                 if not route.exchange and not route.queue and not route.key:
-                    logging.error("Cannot setup AMQ Exchange for '%s', at least one of "
-                                  "Exchange, Queue and/or Key must be present in the routing string. "
-                                  "Expected format is '%s'", route, AMQRoute.FORMAT)
+                    logging_error(
+                        "Cannot setup AMQ Exchange for '%s', at least one of "
+                        "Exchange, Queue and/or Key must be present in the routing string. "
+                        "Expected format is '%s'",
+                        route,
+                        AMQRoute.FORMAT,
+                    )
                 else:
                     if route.exchange != DLQ_EXCHANGE:
                         is_new_route = self.route_database.add_route(route)
                         if is_new_route and self.route_added_notifier:
                             self.route_added_notifier(route)
         except Exception as err:
-            logging.error(" [E] add_routes error: %s", err)
+            logging_error("add_routes error: %s", err)
 
     def set_route_added_notifier(self, callback: Callable[[AMQRoute], None]):
         self.route_added_notifier = callback
 
     def remove_routes(self, message: str) -> None:
         try:
-            to_remove = [AMQRouteFactory.from_string(route_str) for route_str in message.split(",")]
+            to_remove = [
+                AMQRouteFactory.from_string(route_str)
+                for route_str in message.split(",")
+            ]
             removed_count = sum(1 for route in to_remove if self.remove_route(route))
-            print(f" [*] RouterMaster.removeRoutes(): {message}, removed={removed_count}")
+            logging_info(
+                f"RouterMaster.removeRoutes(): {message}, removed={removed_count}"
+            )
         except IOError as err:
-            print(f" [E] Can't remove route(s): {err}")
+            logging_error(f"Can't remove route(s): {err}")
 
     def remove_route(self, route: AMQRoute) -> bool:
         is_route_removed = self.route_database.remove_route(route)

amqp/router/router_consumer.py

@@ -1,143 +1,216 @@
 import logging
 from typing import Set
 
-from pika import spec
-from pika.adapters.blocking_connection import BlockingChannel
-
+from aio_pika.abc import (
+    AbstractRobustChannel,
+    AbstractRobustQueue,
+    AbstractIncomingMessage,
+)
 from amqp.adapter.amq_route_factory import AMQRouteFactory
+from amqp.adapter.logging_utils import (
+    logging_error,
+    logging_info,
+    logging_debug,
+    logging_warning,
+)
 from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import CleverMicroServiceMessage, ServiceMessageType, AMQRoute
-from amqp.router.router_base import CM_C_REQ_QUEUE, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, ANY_RECIPIENT
+from amqp.model.model import ServiceMessage, AMQRoute
+from amqp.model.service_message_type import ServiceMessageType
+from amqp.router.router_base import (
+    CM_C_REQ_QUEUE,
+    CM_REQUEST_EXCHANGE,
+    CM_RESPONSE_EXCHANGE,
+    ANY_RECIPIENT,
+)
 from amqp.router.router_producer import RouterProducer
 
 
 class RouterConsumer(RouterProducer):
     """
-     * Contains Consumer logic. The logic here:
-     * Sets a listener on CM_C_REQ_QUEUE queue to receive routing data requests
-     * Upon receipt of routing data request, responds on CM_RESPONSE_EXCHANGE with local routing data.
-     * It also proactively pushes the local routing data to CM_RESPONSE_EXCHANGE on startup to
-     * notify the other (excluding self) Producers about its presence.
-     *
-     * Consumer is also Producer (Consumer sits alongside the CM Service, and anytime the Service needs
-     * to communicate with other Service, it can do so via the Adapter). Therefore, the Producer
-     * capability is also required.
+    * Contains Consumer logic. The logic here:
+    * Sets a listener on CM_C_REQ_QUEUE queue to receive routing data requests
+    * Upon receipt of routing data request, responds on CM_RESPONSE_EXCHANGE with local routing data.
+    * It also proactively pushes the local routing data to CM_RESPONSE_EXCHANGE on startup to
+    * notify the other (excluding self) Producers about its presence.
+    *
+    * Consumer is also Producer (Consumer sits alongside the CM Service, and anytime the Service needs
+    * to communicate with other Service, it can do so via the Adapter). Therefore, the Producer
+    * capability is also required.
     """
 
     def __init__(self, configuration: AMQConfiguration):
         super().__init__(configuration)
         self.amqRouteFactory = AMQRouteFactory()
-        logging.info(" [*] RouterConsumer.<init>")
+        logging_info("RouterConsumer.<init>")
 
-    def init_routing_paths_consumer(self, _channel: BlockingChannel):
+    async def init_routing_paths_consumer(self, _channel: AbstractRobustChannel):
         """
-         * Initialize router according to Consumer mode.
+        * Initialize router according to Consumer mode.
         """
-
-        self.init_routing_paths(_channel)
-        logging.info(" [*] RouterConsumer.initConsumerRouter, channel open::", self.is_open(_channel))
+        logging_info(
+            "RouterConsumer.initConsumerRouter, channel open: %s",
+            self.is_open(_channel),
+        )
         if self.is_open(_channel):
             try:
-                #  1. declare a Queue that gives the RoutingDAta requests
-                cm_request_queue: str = self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
-                self.channel.queue_declare(cm_request_queue, durable=True, exclusive=False, auto_delete=False)
-                # 1a. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
-                self.channel.queue_bind(queue=cm_request_queue, exchange=CM_REQUEST_EXCHANGE, routing_key="")
+                #  1. declare a Queue to receive the RoutingData requests
+                _cm_request_queue: str = (
+                    self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
+                )
+                _queue: AbstractRobustQueue = await self.channel.declare_queue(
+                    name=_cm_request_queue,
+                    durable=True,
+                    exclusive=False,
+                    auto_delete=False,
+                )
+                # 1a. bind queue to CM_REQUEST_EXCHANGE w/o any conditions (empty routing key)
+                await _queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="")
                 # 1b. Service adapter mode - listens on Request queue and replies to Response queue
-                logging.info(" [*] RouterConsumer listens for RouteDataRequests on:", cm_request_queue)
-                self.channel.basic_consume(queue=cm_request_queue,
-                                           auto_ack=False,
-                                           on_message_callback=self.handle_routing_data_request_message)
+                _c_tag = await _queue.consume(
+                    callback=self.handle_routing_data_request_message, no_ack=False
+                )
+                logging_info(
+                    " [%s] RouterConsumer listens for RouteDataRequests on: %s",
+                    _c_tag,
+                    _cm_request_queue,
+                )
             except Exception as ioe:
-                logging.error(" [E] RouterConsumer FATAL: cannot initialize, %s", ioe)
+                logging_error("RouterConsumer FATAL: cannot initialize, %s", ioe)
         else:
-            logging.error(" [E] RouterConsumer FATAL: cannot initialize, RabbitMQ channel is not open.")
+            logging_error(
+                "RouterConsumer FATAL: cannot initialize, RabbitMQ channel is not open."
+            )
 
     """
      * ServiceMessageType.ROUTING_DATA_REQUEST  handler  listening on queue 'CleverMicroRequest'
     """
 
-    def handle_routing_data_request_message(self,
-                                            ch: BlockingChannel,
-                                            method: spec.Basic.Deliver,
-                                            properties: spec.BasicProperties,
-                                            body: bytes):
+    async def handle_routing_data_request_message(
+        self, message: AbstractIncomingMessage
+    ):
         # some debug statements, TODO - remove for prod release
-        delivery_tag = method.delivery_tag
-        debug = "Delivery.Properties = " + properties.__str__()
-        logging.info(" [DBG] ******[%] (ROUTING_DATA_REQ) tag: %s, env: %s, props: %s",
-                     delivery_tag, method.consumer_tag, method.get_properties(), debug)
+        delivery_tag = message.delivery_tag
+        debug = "Delivery.Properties = " + message.properties.__str__()
+        logging_debug(
+            " ******[%s] (ROUTING_DATA_REQ) tag: %s, env: %s, props: %s",
+            delivery_tag,
+            message.consumer_tag,
+            message.properties,
+            debug,
+        )
 
-        self.channel.basic_ack(delivery_tag, False)
+        await message.ack(False)
 
-        message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(body)
+        cm_message: ServiceMessage = self.service_message_factory.from_bytes(
+            message.body
+        )
         route_mapping: str = self.wrap_own_routes()
-        logging.info(" [DBG] ******[%s] (ROUTING_DATA_REQ) msg=%, routeMap=[{]",
-                     delivery_tag, self.service_message_factory.to_string(message), route_mapping)
+        logging_debug(
+            "******[%s] (ROUTING_DATA_REQ) msg=%s, routeMap=[%s]",
+            delivery_tag,
+            self.service_message_factory.to_string(cm_message),
+            route_mapping,
+        )
         # Reply only if self adapter has own routeMapping (e.g. API-gateway-side adapter has none)
-        if message.is_valid() and message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST and route_mapping:
-            try:
-                service_message: CleverMicroServiceMessage = self.service_message_factory.of(
-                    ServiceMessageType.ROUTING_DATA, route_mapping, message.reply_to
+        if (
+            cm_message.is_valid()
+            and cm_message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST
+            and route_mapping
+        ):
+            if cm_message.reply_to != self.amq_configuration.amq_adapter.service_name:
+                try:
+                    service_message: ServiceMessage = self.service_message_factory.of(
+                        ServiceMessageType.ROUTING_DATA,
+                        route_mapping,
+                        cm_message.reply_to,
+                    )
+                    await self.publish_service_message(
+                        service_message, self.cm_response_exchange
+                    )
+                except Exception as err:
+                    logging_error(
+                        "******[%s] (ROUTING_DATA_REQ) Response to queue: %s failed: %s",
+                        delivery_tag,
+                        CM_RESPONSE_EXCHANGE,
+                        err,
+                    )
+            else:
+                logging_info(
+                    "******[%s] (ROUTING_DATA_REQ) [%s] Ignore request originated by ourselves.",
+                    delivery_tag,
+                    str(message.body),
                 )
-                self.publish_service_message(service_message, CM_RESPONSE_EXCHANGE)
-            except Exception as err:
-                logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) Response to queue: %s failed: %s",
-                              delivery_tag, CM_RESPONSE_EXCHANGE, err)
-
         else:
-            logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) [%s] produces invalid or no CM msg type.",
-                          delivery_tag, str(body))
+            logging_error(
+                "******[%s] (ROUTING_DATA_REQ) [%s] produces invalid or no CM msg type.",
+                delivery_tag,
+                str(message.body),
+            )
 
-    def register_route(self, route: AMQRoute):
+    async def register_route(self, route: AMQRoute):
         """
-         * Called in Consumer (Service Adapter) context. Registers the routing data for self Service
-         * with the Master router by sending own routing data to Master via CleverMicroResponse queue.
-         * :param route: route to register with Master
+        * Called in Consumer (Service Adapter) context. Registers the routing data for self Service
+        * with the Master router by sending own routing data to Master via CleverMicroResponse queue.
+        * :param route: route to register with Master
         """
         try:
             if self.is_open(self.channel) and route.queue:
-                serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
+                serviceMessage: ServiceMessage = self.service_message_factory.of(
                     ServiceMessageType.ROUTING_DATA, route.as_string(), ANY_RECIPIENT
                 )
-                self.publish_service_message(serviceMessage, CM_RESPONSE_EXCHANGE)
+                await self.publish_service_message(
+                    serviceMessage, self.cm_response_exchange
+                )
                 self.add_own_route(route)
-                logging.info(" [DBG] RouterConsumer registered Route::", route)
+                logging_info("RouterConsumer registered Route: %s", route)
             else:
-                logging.warning(" [W] RouterConsumer couldn't register route data: %s channel isn't open.",
-                                CM_RESPONSE_EXCHANGE)
+                logging_warning(
+                    "RouterConsumer couldn't register route data: %s channel isn't open.",
+                    CM_RESPONSE_EXCHANGE,
+                )
         except Exception as ioe:
-            logging.error(" [E] RouterConsumer failed register route data: %s", ioe)
+            logging_error("RouterConsumer failed register route data: %s", ioe)
 
-    def remove_route_on_cleanup(self, route: AMQRoute) -> bool:
+    async def remove_route_on_cleanup(self, route: AMQRoute) -> bool:
         """
-         * Unregister / remove route from the list
-         * :param route: route to remove
-         * :return result of remove operation
+        * Unregister / remove route from the list
+        * :param route: route to remove
+        * :return result of remove operation
         """
-        logging.info(" [DBG] RouterConsumer.removeRoute, route::", route.as_string())
+        logging_info("RouterConsumer.removeRoute, route: %s", route.as_string())
         try:
-            service_message: CleverMicroServiceMessage = self.service_message_factory.of(
-                ServiceMessageType.ROUTING_DATA_REMOVE,
-                route.as_string(),
-                ANY_RECIPIENT
+            service_message: ServiceMessage = self.service_message_factory.of(
+                ServiceMessageType.ROUTING_DATA_REMOVE, route.as_string(), ANY_RECIPIENT
             )
-            if not self.publish_service_message(service_message, CM_RESPONSE_EXCHANGE):
-                logging.warning(" [W] RouterConsumer couldn't remove current route: %s, channel not open.",
-                                route)
+            if not await self.publish_service_message(
+                service_message, self.cm_response_exchange
+            ):
+                logging_warning(
+                    "RouterConsumer couldn't remove current route: %s, channel not open.",
+                    route,
+                )
 
         except Exception as ioe:
-            logging.error(" [E] RouterConsumer failed remove current route: %s", ioe)
+            logging_error("RouterConsumer failed remove current route: %s", ioe)
 
         result: bool = self.remove_route(route)
         return result
 
     def cleanup(self):
         """
-         * Cleanup - de-register routes with master.
+        * Cleanup - de-register routes with master.
         """
         routes: Set[AMQRoute] = self.get_routes()
-        logging.info(" [DBG] RouterConsumer.cleanup() [%d]: %s", len(routes), self.wrap_routes())
-        not_removed = sum(1 for r in (self.remove_route_on_cleanup(route) for route in routes) if not r)
+        logging_info(
+            "RouterConsumer.cleanup() [%d]: %s", len(routes), self.wrap_routes()
+        )
+        not_removed = sum(
+            1
+            for r in (self.remove_route_on_cleanup(route) for route in routes)
+            if not r
+        )
         if not_removed > 0:
-            logging.info(" [W] %d route(s) not removed. self may lead to failed delivery on self route", not_removed)
+            logging_warning(
+                "%d route(s) not removed. self may lead to failed delivery on self route",
+                not_removed,
+            )

amqp/router/router_producer.py

@@ -1,190 +1,237 @@
 """
- * Maps the request to RabbitMQ exchange. Handles the service discovery logic.
- *
- * Router can be either in 'Master' or 'Service (Consumer)' modes. 'Master' mode is the outward
- * facing side that receives the requests and routes it to queue associated with requested service.
- * 'Service' mode is consumer side for given service, which receives the message from AMQ and
- * forwards it to the application.
- """
-import logging
-import os
-import pika
-from pika import BasicProperties, spec
-from pika.adapters.blocking_connection import BlockingChannel
-from pika.exchange_type import ExchangeType
+* Maps the request to RabbitMQ exchange. Handles the service discovery logic.
+*
+* Router can be either in 'Master' or 'Service (Consumer)' modes. 'Master' mode is the outward
+* facing side that receives the requests and routes it to queue associated with requested service.
+* 'Service' mode is consumer side for given service, which receives the message from AMQ and
+* forwards it to the application.
+"""
 
-from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
+import logging
+
+from aio_pika import Message
+from aio_pika.abc import (
+    AbstractRobustChannel,
+    AbstractRobustQueue,
+    AbstractRobustConnection,
+    AbstractRobustExchange,
+    AbstractIncomingMessage, ExchangeType,
+)
+
+from amqp.adapter.logging_utils import (
+    logging_error,
+    logging_info,
+    logging_debug,
+    logging_warning,
+)
+from amqp.adapter.service_message_factory import ServiceMessageFactory
 from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import CleverMicroServiceMessage, ServiceMessageType
-from amqp.router.router_base import RouterBase, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, CM_C_AMQ_QUEUE, \
-    CM_P_RESP_QUEUE, CM_P_REPLY_TO, ANY_RECIPIENT
+from amqp.model.model import ServiceMessage
+from amqp.model.service_message_type import ServiceMessageType
+from amqp.router.router_base import (
+    RouterBase,
+    CM_REQUEST_EXCHANGE,
+    CM_RESPONSE_EXCHANGE,
+    CM_C_AMQ_QUEUE,
+    CM_P_RESP_QUEUE,
+    CM_P_REPLY_TO,
+    ANY_RECIPIENT,
+)
 
 
 class RouterProducer(RouterBase):
 
     def __init__(self, configuration: AMQConfiguration):
         """
-         * Initialize router - need to supply the config values and RabbitMQ channel.
-         * :param configuration:  adapter's configuration reference.
-         """
-        super().__init__()
-        self.connection = None
-        self.service_message_factory = CleverMicroServiceMessageFactory(configuration.amq_adapter.service_name)
-        self.amq_configuration: AMQConfiguration = configuration
-        self.channel: pika.adapters.blocking_connection.BlockingChannel | None = None
-
-        logging.info(f" [*] RouterProducer.<init>: %s, route: %s",
-                     self.amq_configuration.amq_adapter.service_name,
-                     self.amq_configuration.amq_adapter.route_mapping)
-
-    def amq_consumer(self):
-        credentials = pika.PlainCredentials(
-            os.environ['RABBIT_MQ_USER'], os.environ['RABBIT_MQ_PASSWORD']
-        )
-        self.connection = pika.BlockingConnection(
-            pika.ConnectionParameters(host='localhost', credentials=credentials)
-        )
-        self.channel = self.connection.channel()
-        self.channel.start_consuming()
-
-    def init_routing_paths(self, channel: pika.adapters.blocking_connection.BlockingChannel | None):
+        * Initialize router - need to supply the config values and RabbitMQ channel.
+        * :param configuration:  adapter's configuration reference.
+        """
+        super().__init__()
+        self.connection: AbstractRobustConnection | None = None
+        self.service_message_factory = ServiceMessageFactory(
+            configuration.amq_adapter.service_name
+        )
+        self.amq_configuration: AMQConfiguration = configuration
+        self.channel: AbstractRobustChannel | None = None
+        self.cm_request_exchange: AbstractRobustExchange | None = None
+        self.cm_response_exchange: AbstractRobustExchange | None = None
+
+        logging_info(
+            f"RouterProducer.<init>: %s, route: %s",
+            self.amq_configuration.amq_adapter.service_name,
+            self.amq_configuration.amq_adapter.route_mapping,
+        )
+
+    async def init_internal_routing_paths(self, channel: AbstractRobustChannel | None):
+        """
+        * Initialize internal routes for ServiceMessages.
+        * The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
+        * created to enable the service discovery.
+        * The Producer mode sends out routing data request on CleverMicroRequest and listens
+        * on CleverMicroResponse for any service responding with its routing data.
         """
-         * Initialize router according to Producer mode.
-         * The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
-         * created to enable the service discovery.
-         * The Producer mode sends out routing data request on CleverMicroRequest and listens
-         * on CleverMicroResponse for any service responding with its routing data.
-         """
 
         self.channel = channel
-        logging.info(" [*] RouterProducer.initRoutingPaths, channel open: %s", self.is_open(self.channel))
+        logging_info(
+            "RouterProducer.initRoutingPaths, channel open: %s",
+            self.is_open(self.channel),
+        )
         if self.is_open(self.channel):
             try:
                 # **** set up the service discovery internal exchanges ****
                 # 1. declare RequestExchange to where to publish the RoutingData requests
-                self.channel.exchange_declare(exchange=CM_REQUEST_EXCHANGE, exchange_type=ExchangeType.fanout)
+                self.cm_request_exchange = await self.channel.declare_exchange(
+                    name=CM_REQUEST_EXCHANGE, type=ExchangeType.FANOUT
+                )
                 #
                 # 2. declare ResponseExchange where to publish current RoutingData (response to
                 #    a RoutingData request)
-                self.channel.exchange_declare(exchange=CM_RESPONSE_EXCHANGE, exchange_type=ExchangeType.fanout)
+                self.cm_response_exchange = await self.channel.declare_exchange(
+                    name=CM_RESPONSE_EXCHANGE, type=ExchangeType.FANOUT
+                )
                 # 2a. declare a Response queue for Routing Data replies from Services
-                cm_response_queue: str = self.get_response_queue_name()
-                self.channel.queue_declare(cm_response_queue, durable=True, exclusive=False, auto_delete=False,
-                                           arguments={})
+                _cm_response_queue: str = self.get_response_queue_name()
+                _response_queue: AbstractRobustQueue = await self.channel.declare_queue(
+                    name=_cm_response_queue,
+                    durable=True,
+                    exclusive=False,
+                    auto_delete=False,
+                    arguments={},
+                )
                 # 2b. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
-                self.channel.queue_bind(queue=cm_response_queue, exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
+                await _response_queue.bind(
+                    exchange=CM_RESPONSE_EXCHANGE, routing_key="#"
+                )
                 # 2c. listen for incoming RoutingData messages
-                self.channel.basic_consume(
-                    queue=cm_response_queue,
-                    auto_ack=False,
-                    on_message_callback=self.handle_routing_data_message)
-                self.channel.start_consuming()
-                logging.info(" [*] Routing master listens for Route updates on %s", cm_response_queue)
+                _c_tag = await _response_queue.consume(
+                    callback=self.handle_routing_data_message, no_ack=False
+                )
+                logging_info(
+                    " [%s] Routing master listens for Route updates on %s",
+                    _c_tag,
+                    _cm_response_queue,
+                )
             except Exception as ioe:
-                logging.error(" [E] RouterProducer FATAL: cannot initialize, %s", ioe)
+                logging_error("RouterProducer FATAL: cannot initialize, %s", ioe)
 
         else:
-            logging.error(" [E] RouterProducer FATAL: cannot initialize, RabbitMQ channel is not open.")
+            logging_error(
+                "RouterProducer FATAL: cannot initialize, RabbitMQ channel is not open."
+            )
 
-    def handle_routing_data_message(self,
-                                    ch: BlockingChannel,
-                                    method: spec.Basic.Deliver,
-                                    properties: spec.BasicProperties,
-                                    body: bytes):
+    async def handle_routing_data_message(self, message: AbstractIncomingMessage):
         """
-         * Handler for CLEVER_MICRO_RESPONSE queue messages. Used in Master mode to handle receipt of
-         * routing data, invokes addRoute or removeRoute depending on the type of the message.
-         """
-        print(f" [x] Received {body}, m={method}, p={properties}, ch={ch}")
+        * Handler for CLEVER_MICRO_RESPONSE queue messages. Used in Master mode to handle receipt of
+        * routing data, invokes addRoute or removeRoute depending on the type of the message.
         """
-        logging.info(" [DBG] ******[%s] ROUTING_DATA %s >>> %s",
-              delivery.getEnvelope().getDeliveryTag(),
-              delivery.getEnvelope().toString(), str(delivery.getBody()))
-              """
-        # ACK message now so that it is not being redelivered
-        self.channel.basic_ack(method.delivery_tag, False)
-        message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(body)
+        logging_info(
+            f"[%s] Received ROUTING DATA: [%s], msgId=%s",
+            message.delivery_tag,
+            message.body,
+            message.message_id,
+        )
+        await message.ack(multiple=False)
+        cm_message: ServiceMessage = self.service_message_factory.from_bytes(
+            message.body
+        )
         # ignore the message if it comes from ourselves (even if from different instance)
-        logging.info(" [DBG] ******[%s] ROUTING_DATA, msg=%s, routeMap=[{] sn={, replyTo={",
-                     method.delivery_tag,
-                     self.service_message_factory.to_string(message),
-                     self.amq_configuration.amq_adapter.route_mapping,
-                     self.amq_configuration.amq_adapter.service_name,
-                     message.reply_to)
-        if message.is_valid() and not self.amq_configuration.amq_adapter.service_name == message.reply_to:
-            if message.message_type == ServiceMessageType.ROUTING_DATA:
-                self.add_routes(message.message)
-            if message.message_type == ServiceMessageType.ROUTING_DATA_REMOVE:
-                self.remove_routes(message.message)
+        logging_debug(
+            "******[%s] ROUTING_DATA, msg=%s, routeMap=[%s] sn=%s, replyTo=%s",
+            message.delivery_tag,
+            self.service_message_factory.to_string(cm_message),
+            self.amq_configuration.amq_adapter.route_mapping,
+            self.amq_configuration.amq_adapter.service_name,
+            cm_message.reply_to,
+        )
+        if (
+                cm_message.is_valid()
+                and not self.amq_configuration.amq_adapter.service_name
+                        == cm_message.reply_to
+        ):
+            if cm_message.message_type == ServiceMessageType.ROUTING_DATA:
+                self.add_routes(cm_message.message)
+            if cm_message.message_type == ServiceMessageType.ROUTING_DATA_REMOVE:
+                self.remove_routes(cm_message.message)
 
-    def request_routes_from_adapters(self):
+    async def request_routes_from_adapters(self):
         """
-         * Send a broadcast message prompting all listening adapters to reply with own routing data.
-         """
-        logging.info(" [*] RouterProducer.requestRoutesFromAdapters, queue name: %s", CM_REQUEST_EXCHANGE)
+        * Send a broadcast message prompting all listening adapters to reply with own routing data.
+        """
+        logging_info(
+            "RouterProducer.requestRoutesFromAdapters, queue name: %s",
+            CM_REQUEST_EXCHANGE,
+        )
         try:
-            serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
+            serviceMessage: ServiceMessage = self.service_message_factory.of(
                 ServiceMessageType.ROUTING_DATA_REQUEST, "", ANY_RECIPIENT
             )
-            if not self.publish_service_message(serviceMessage, CM_REQUEST_EXCHANGE):
-                logging.warning(" [E] RouterProducer could not request route data: %s channel is not open.",
-                                CM_REQUEST_EXCHANGE)
+            if not await self.publish_service_message(
+                    serviceMessage, self.cm_request_exchange
+            ):
+                logging_warning(
+                    "RouterProducer could not request route data: %s channel is not open.",
+                    CM_REQUEST_EXCHANGE,
+                )
 
         except Exception as ioe:
-            logging.error(" [E] RouterProducer failed request routing data: %s", ioe)
+            logging_error("RouterProducer failed request routing data: %s", ioe)
 
-    BASIC: BasicProperties = BasicProperties(content_type="application/octet-stream",
-                                             content_encoding=None,
-                                             headers=None,
-                                             delivery_mode=1,
-                                             priority=0,
-                                             correlation_id=None)
-
-    def publish_service_message(self, message: CleverMicroServiceMessage, exchange_name: str) -> bool:
+    async def publish_service_message(
+            self, message: ServiceMessage, exchange: AbstractRobustExchange
+    ) -> bool:
+        """
+        * Publishes a service message to the service queue and logs success log entry.
+        *
+        * :param message:   Service Message to be sent
+        * :param exchange: target exchange name where to publish the message
+        * :return True if channel is open, False otherwise
+        * @throws IOException if something else goes wrong
         """
-         * Publishes a service message to the service queue and logs success log entry.
-         *
-         * :param message:   Service Message to be sent
-         * :param exchangeName: target exchange name where to publish the message
-         * :param queueName target: queue name (Optional, purpose is to get the queue size only)
-         * :return True if channel is open, False otherwise
-         * @throws IOException if something else goes wrong
-         """
         if self.is_open(self.channel):
             binary_content: bytes = self.service_message_factory.serialize(message)
-            self.channel.basic_publish(exchange=exchange_name,
-                                       routing_key="",  # empty routing key
-                                       properties=self.BASIC,
-                                       body=binary_content)
-            logging.info(" [x] Service Message Sent to %s, msg: %s", exchange_name, str(binary_content))
+            pika_message: Message = Message(
+                body=binary_content,
+                content_encoding="utf-8",
+                delivery_mode=2,
+                content_type="application/octet-stream",
+                headers=None,
+                priority=0,
+                correlation_id=None,
+            )
+            await exchange.publish(message=pika_message, routing_key="")
+            logging_info(
+                "Service Message Published to %s, msg: %s",
+                exchange.name,
+                str(binary_content),
+            )
             return True
         return False
 
     def get_reply_to_queue_name(self) -> str:
         """
-         * self should create a unique name for self process and also when there's several instances.
-         * @return unique reply-to queue name.
-         """
+        * self should create a unique name for self process and also when there's several instances.
+        * @return unique reply-to queue name.
+        """
         return self.amq_configuration.amq_adapter.adapter_prefix() + CM_P_REPLY_TO
 
     def get_response_queue_name(self) -> str:
         """
-         * self should create a unique name for self process and also when there's several instances.
-         * @return unique reply-to queue name.
-         """
+        * self should create a unique name for self process and also when there's several instances.
+        * @return unique reply-to queue name.
+        """
         return self.amq_configuration.amq_adapter.adapter_prefix() + CM_P_RESP_QUEUE
 
     def get_consume_queue_name(self) -> str:
         """
-         * self should create a unique name for self process and also when there's several instances.
-         * @return unique reply-to queue name.
-         """
+        * self should create a unique name for self process and also when there's several instances.
+        * @return unique reply-to queue name.
+        """
         return self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_AMQ_QUEUE
 
-    def is_open(self, _channel: pika.adapters.blocking_connection.BlockingChannel) -> bool:
+    def is_open(self, _channel: AbstractRobustChannel) -> bool:
         """
-         * Test if the channel is usable (opened).
-         * :param _channel: the channel
-         * :return True if the channel is opened.
-         """
-        return _channel is not None and _channel.is_open
+        * Test if the channel is usable (opened).
+        * :param _channel: the channel
+        * :return True if the channel is opened.
+        """
+        return _channel is not None and not _channel.is_closed

amqp/router/test_route_database.py

@@ -1,24 +0,0 @@
-from unittest import TestCase
-
-
-class TestRouteDatabase(TestCase):
-    def test_get_routes(self):
-        self.fail()
-
-    def test_pattern(self):
-        self.fail()
-
-    def test_matches(self):
-        self.fail()
-
-    def test_wrap_routes(self):
-        self.fail()
-
-    def test_find_route(self):
-        self.fail()
-
-    def test_add_route(self):
-        self.fail()
-
-    def test_remove_route(self):
-        self.fail()

amqp/router/test_router_base.py

@@ -1,39 +0,0 @@
-from unittest import TestCase
-
-
-class TestRouterBase(TestCase):
-    def test_get_routing_key(self):
-        self.fail()
-
-    def test_wrap_routes(self):
-        self.fail()
-
-    def test_wrap_own_routes(self):
-        self.fail()
-
-    def test_unwrap_route_list(self):
-        self.fail()
-
-    def test_get_routes(self):
-        self.fail()
-
-    def test_find_route(self):
-        self.fail()
-
-    def test_find_route_by_message(self):
-        self.fail()
-
-    def test_add_routes(self):
-        self.fail()
-
-    def test_remove_routes(self):
-        self.fail()
-
-    def test_remove_route(self):
-        self.fail()
-
-    def test_add_route(self):
-        self.fail()
-
-    def test_add_own_route(self):
-        self.fail()

amqp/router/utils.py

@@ -1,9 +1,37 @@
+import asyncio
 import re
 
+pattern = re.compile(r"[^a-zA-Z0-9.#/\-]")
+AWAIT_SLEEP = 0.05  # seconds
+
 
 def sanitize_as_rabbitmq_name(input_str: str) -> str:
-    pattern = re.compile(r"[^a-zA-Z0-9.#/\-]")
     matcher = pattern.search(input_str)
-    token = input_str[:matcher.start()] if matcher else input_str
-    step1 = re.sub(r"[^A-Za-z0-9]", ".", token)
-    return re.sub(r"\.+", ".", step1)
+    token = input_str[: matcher.start()] if matcher else input_str
+    return sanitize_as_rabbitmq_domain_name(token)
+
+
+def sanitize_as_rabbitmq_domain_name(token: str) -> str:
+    # Step 1: Replace non-alphanumeric characters (except periods) with a period
+    step1 = re.sub(r"[^A-Za-z0-9.#]", ".", token)
+    # Step 2: Replace multiple consecutive periods with a single period
+    step2 = re.sub(r"\.+", ".", step1)
+    # Step 3: Remove trailing '.' character, if applicable
+    step_last_char = len(step2) - 1
+    return step2[:-1] if step_last_char > 0 and step2[step_last_char] == "." else step2
+
+
+async def await_future(future: asyncio.Future, sleep_time=AWAIT_SLEEP):
+    """
+    Wait for a future to complete.
+    """
+    while not future.done():
+        await asyncio.sleep(sleep_time)
+
+
+async def await_result(future: asyncio.Future, sleep_time=AWAIT_SLEEP):
+    """
+    Wait for a future to complete and return its result.
+    """
+    await await_future(future, sleep_time=sleep_time)
+    return future.result()

amqp/service/amq_message_handler.py

@@ -0,0 +1,286 @@
+import asyncio
+import threading
+import time
+import os
+from asyncio import Future, AbstractEventLoop
+from typing import Dict
+
+from aio_pika import Message
+from aio_pika.abc import AbstractIncomingMessage, AbstractRobustExchange
+
+from opentelemetry import trace
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+from opentelemetry.trace import Tracer, TraceState
+
+from amqp.adapter.backpressure_handler import BackpressureHandler
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.data_response_factory import DataResponseFactory
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, AMQMessage
+from amqp.adapter.file_handler import StreamingFileHandler, FileHandler
+from amqp.adapter.logging_utils import logging_error, logging_debug, logging_info
+from amqp.adapter.service_message_factory import ServiceMessageFactory
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import (
+    DataResponse,
+    DataMessage,
+    MultipartDataMessage,
+)
+from amqp.rabbitmq.rabbit_mq_client import RabbitMQClient
+from amqp.adapter.serializer import map_as_string
+
+
+def collect_trace_info():
+    trace_map = {}
+    TraceContextTextMapPropagator().inject(
+        carrier=trace_map, context=trace.context_api.get_current()
+    )
+    return trace_map
+
+
+def get_default_trace_state():
+    return TraceState(entries={})
+
+
+def set_text(carrier, key, value):
+    carrier[key] = value
+
+
+class DataMessageHandler:
+    def __init__(
+        self,
+        service_adapter: CleverThisServiceAdapter,
+        tracer: Tracer,
+        message_factory: DataMessageFactory,
+        service_message_factory: ServiceMessageFactory,
+        reply_to_exchange: AbstractRobustExchange,
+        rabbit_mq_client: RabbitMQClient,
+        amq_configuration: AMQConfiguration,
+        loop,
+        backpressure_handler: BackpressureHandler,
+        file_handler: FileHandler = StreamingFileHandler(),
+    ):
+        self.service_adapter: CleverThisServiceAdapter = service_adapter
+        self.tracer = tracer
+        self.message_factory: DataMessageFactory = message_factory
+        self.service_message_factory: ServiceMessageFactory = service_message_factory
+        self.reply_to_exchange: AbstractRobustExchange = reply_to_exchange
+        self.rabbit_mq_client: RabbitMQClient = rabbit_mq_client
+        self.rabbit_mq_url = amq_configuration.dispatch.rabbit_mq_url
+        self.output_dir = amq_configuration.dispatch.download_dir
+        self.loop = loop
+        self.service_adapter.loop = loop
+        # Dictionary to store outstanding Futures
+        self.outstanding: Dict[str, asyncio.Future] = {}
+        self.reply_to: str | None = None
+        self.file_handler: FileHandler = file_handler
+        self.backpressure_handler: BackpressureHandler = backpressure_handler
+
+    async def inbound_data_message_callback(self, message: AbstractIncomingMessage):
+        threading.Thread(
+            target=lambda: asyncio.run(self.run_callback_thread(message))
+        ).start()
+
+    async def run_callback_thread(self, message: AbstractIncomingMessage):
+        """
+        Receives the DataMessage as bytes from RabbitMQ queue and deserializes it to the DataMessage object
+        Ensures Jaeger trace-id propagation.
+        Invokes custom handler/mapper/adapter method that corresponds to this message
+        :param message:
+        :return:
+        """
+        start = time.time()
+        amq_message = await self.reconstitute_data_message(message)
+        if amq_message is not None:
+            self.ensure_trace_info(amq_message)
+            if amq_message.magic == DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST:
+                await self.run_http_request_magic(message, amq_message, self.loop)
+            else:
+                logging_error(
+                    "Unknown MAGIC value: [%s], don't know how to interpret the message",
+                    amq_message.magic,
+                )
+                asyncio.run_coroutine_threadsafe(message.nack(requeue=False), self.loop)
+                return
+        else:
+            logging_error(
+                "######[%s] No valid AMQ Message present.", message.delivery_tag
+            )
+        self.record_duration(start, message.delivery_tag)
+        asyncio.run_coroutine_threadsafe(message.ack(multiple=False), self.loop)
+
+    async def run_http_request_magic(
+        self,
+        message: AbstractIncomingMessage,
+        amq_message: DataMessage,
+        loop: AbstractEventLoop,
+    ) -> None:
+        """
+        Synchronous version of on_message method. This is a wrapper around the async on_message method.
+        It also counts parallel executions and handles backpressure.
+        :param message: RabbitMQ raw message
+        :param amq_message: DataMessage
+        :param loop: Event loop
+        :return: DataResponse
+        """
+        # Increment the counter for parallel executions
+        self.backpressure_handler.increase_parallel_executions()
+        logging_info(
+            f"PARALLEL: Current Capacity [{self.backpressure_handler.current_parallel_executions}] of [{self.backpressure_handler.parallel_workers}]"
+        )
+        if message.reply_to:
+            self.reply_to = message.reply_to
+
+        try:
+            await self.backpressure_handler.check_overload_condition()
+            _a_message = AMQMessage(amq_message, self.rabbit_mq_client.connection)
+            _response: DataResponse = await self.service_adapter.on_message(_a_message)
+            logging_debug(f"Result in thread: {_response}")
+            try:
+                await self.send_reply(message.reply_to, _response)
+            except Exception as e:
+                logging_error(f"Processing message: {e}")
+                asyncio.run_coroutine_threadsafe(message.nack(requeue=False), loop=loop)
+
+            if isinstance(amq_message, MultipartDataMessage):
+                # remove the downloaded files from the output_dir
+                for file_id, filename in amq_message.extra_data.items():
+                    try:
+                        logging_info(
+                            f"[{message.delivery_tag}] Deleting file: {file_id} in {filename}"
+                        )
+                        os.remove(filename)
+                    except OSError as e:
+                        logging_error(f"Deleting file {filename}: {e}")
+        except Exception as e:
+            logging_error(f"Request handler: {e}")
+
+        self.backpressure_handler.decrease_parallel_executions()
+
+    async def reconstitute_data_message(
+        self, message: AbstractIncomingMessage
+    ) -> DataMessage | None:
+        amq_message: DataMessage | None = None
+        delivery_tag = message.delivery_tag
+        addressing: int = message.headers.get("clevermicro.addressing", 0)
+        if addressing == 0:
+            amq_message = DataMessageFactory.from_bytes(message.body)
+        else:
+            amq_message = await DataMessageFactory.from_stream(
+                message.body, rabbitmq_url=self.rabbit_mq_url, loop=self.loop
+            )
+            logging_info(
+                f"######[{delivery_tag}] AMQ Message from stream: {amq_message}"
+            )
+            if amq_message is not None:
+                extra_data = await self.file_handler.on_message(
+                    message=message,
+                    json_data=amq_message.body(),
+                    rabbitmq_url=self.rabbit_mq_url,
+                    loop=self.loop,
+                    output_dir=self.output_dir,
+                )
+                logging_info(
+                    f"######[{delivery_tag}] Multipart download result: {extra_data}"
+                )
+                amq_message = MultipartDataMessage(amq_message, extra_data)
+        if amq_message:
+            logging_info(
+                "######[%s] AMQ Message received, ctag=%s, msgId=%s, %s:%s, traceInfo=%s",
+                message.delivery_tag,
+                message.consumer_tag,
+                amq_message.id,
+                amq_message.method,
+                amq_message.path,
+                map_as_string(amq_message.trace_info),
+            )
+        return amq_message
+
+    def ensure_trace_info(self, amq_message: DataMessage):
+        propagator = TraceContextTextMapPropagator()
+        context = propagator.extract(
+            carrier=amq_message.trace_info, context=trace.context_api.get_current()
+        )
+        self.tracer.start_span(name="AMQ-Adapter-Consumer", context=context)
+        # logging_info("CTX=%s", context)
+        # current_span = self.tracer.start_span("AMQ-Adapter.Processing.Request", parent=parent_span)
+        # context_with_span = trace.set_span_in_context(current_span, context)
+        # logging.info(" [*] CTX2=%s", context_with_span)
+        # propagator.inject(context_with_span, amq_message.trace_info, self.set_text)
+        # logging.info(" [***] NEW TI=%s CTX=%s SCOPE=%s",
+        #             map_as_string(amq_message.trace_info), context_with_span, context)
+
+    def record_duration(self, start, delivery_tag):
+        global last_data_message_time
+        last_data_message_time = time.time()
+        logging_info(
+            "######[%s] Done, single ACK: %sms.",
+            delivery_tag,
+            last_data_message_time - start,
+        )
+
+    async def send_reply(self, reply_to: str, response: DataResponse):
+        """
+        The Service side code. When the CleverXXX service has output ready, in DataResponse object,
+        call this method to return the reply back to the caller
+        :param reply_to:connection.channel
+        :param response:
+        :return:
+        """
+        if reply_to:
+            pika_message: Message = Message(
+                body=DataResponseFactory.serialize(response),
+                correlation_id=response.id,
+                content_type=(
+                    response.content_type[0]
+                    if isinstance(response.content_type, list)
+                    else response.content_type
+                ),
+            )
+            _res = asyncio.run_coroutine_threadsafe(
+                self.reply_to_exchange.publish(
+                    message=pika_message, routing_key=reply_to
+                ),
+                self.loop,
+            )
+            logging_debug(
+                "###### DONE Reply Published to exchg:(%s) To=%s, res=%s",
+                self.reply_to_exchange.name,
+                reply_to,
+                _res,
+            )
+
+    async def reply_received_callback(self, message: AbstractIncomingMessage):
+        """
+        The reply for DataMessage that we sent out earlier - must match the received response with
+        the original request and respond to the caller.
+        :param message:
+        :return:
+        """
+        reply_id = message.correlation_id
+        content_type = message.content_type
+        delivery_tag = message.delivery_tag
+        debug = f"######[{delivery_tag}] (RPC REPLY) env: {message.headers}, props: {message.properties}, BODY: {message.body.decode()}"
+        logging_debug(debug)
+        # TODO - figure out implementation of this part, as this is sending reply back either to a service,
+        #        or to the user
+        logging_info(
+            f"OUTSTANDING RECV: {self.outstanding}, size={len(self.outstanding)}"
+        )
+
+        future: Future = self.outstanding.pop(reply_id, None)
+        # if reply is None:
+        # logging.warn(" [W] ######[%s] IGNORING REPLY. No outstanding request for ID %s",
+        #             delivery_tag, reply_id)
+        # else:
+        # result = DataResponseFactory.from_bytes(body)
+        # reply.tracing_span().end()
+        # TODO - figure out the right response format (not needed for Clever Swarm v0.1)
+        # that currently runs Java version, this method here is for the future compatibility to enable
+        # internal, service-2-service communication, and the proper response type will be decided than
+        # reply.response().set_result(Response.ok(result.body, content_type).build())
+        if future:
+            response: DataResponse = DataResponseFactory.from_bytes(message.body)
+            # Complete the Future with the response data
+            future.set_result(response.body())
+
+        await message.ack(multiple=False)

amqp/service/amq_service.py

@@ -1,84 +1,128 @@
-import os
 import logging
-from functools import partial
-from opentelemetry import trace
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import (
-    ConsoleSpanExporter,
-    SimpleSpanProcessor
-)
-from opentelemetry.propagate import set_global_textmap
-from opentelemetry.propagators.tracecontext import TraceContextTextMapPropagator
+import logging.config
+import os
+import asyncio
+from asyncio import Future
+from threading import Thread
+from typing import Optional
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
+from aio_pika.abc import AbstractRobustExchange
+
+from amqp.adapter.backpressure_handler import BackpressureHandler
+from amqp.adapter.data_message_factory import DataMessageFactory
 from amqp.adapter.amq_route_factory import AMQRouteFactory
-from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
-from amqp.rabbitmq.rabbit_mq_consumer import RabbitMQConsumer
-from amqp.service.service_message_handler import AMQServiceMessageHandler
+from amqp.adapter.logging_utils import logging_info, logging_warning
+from amqp.adapter.service_message_factory import ServiceMessageFactory
+
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import DataMessage
+from amqp.rabbitmq.rabbit_mq_client import RabbitMQClient
+from amqp.service.amq_message_handler import DataMessageHandler
+from amqp.service.tracing import initialize_trace
+
+
+logging.basicConfig(level=logging.DEBUG)
+logging.info(f"CWD = {os.getcwd()}")
+logging_conf_path = os.path.join(os.getcwd(), "amqp", "service", "logging.conf")
+if os.path.exists(logging_conf_path):
+    logging.config.fileConfig(logging_conf_path)
+else:
+    logging_conf_path = os.path.join(os.getcwd(), "logging.conf")
+    if os.path.exists(logging_conf_path):
+        logging.config.fileConfig(logging_conf_path)
 
 
 class AMQService:
     CLEVER_AMQP_CLIENT_VERSION = os.getenv("CLEVER_AMQP_CLIENT_VERSION")
     MAX_WAIT = 300  # defaults to 5 minutes (300 seconds)
 
-    def __init__(self, amq_configuration, http_adapter):
-        self.amq_configuration = amq_configuration
-        self.http_adapter = http_adapter
-        self.message_factory = AMQMessageFactory.get_instance(self.amq_configuration.amq_adapter().generator_id())
-        self.rabbit_mq_consumer = RabbitMQConsumer(self.amq_configuration)
-        self.amq_service_message_handler = AMQServiceMessageHandler(self.http_adapter, self.get_tracer())
-
-    def init(self):
-        CleverMicroServiceMessageFactory.process_name(self.amq_configuration.amq_adapter().service_name())
-        logging.info("*********  AMQService.isRouter = %s ***********", self.amq_configuration.amq_adapter().is_router())
+    def __init__(self, amq_configuration: AMQConfiguration, service_adapter=None):
         logging.info("***********************************************")
         logging.info("**********  AMQ   Service   *******************")
         logging.info("***********************************************")
+        self.loop = asyncio.new_event_loop()
 
-        self.setup_opentelemetry()
-        self.register_routes()
-        self.register_rpc_handler()
-        self.rabbit_mq_consumer.request_routes()
+        self.amq_configuration = amq_configuration
+        self.service_adapter = service_adapter
+        self.data_message_factory = DataMessageFactory.get_instance(
+            self.amq_configuration.amq_adapter.generator_id
+        )
+        self.rabbit_mq_client = RabbitMQClient(self.amq_configuration)
+        self.tracer = initialize_trace(name=amq_configuration.amq_adapter.service_name)
+        self.amq_data_message_handler = None
+        self.service_message_factory = ServiceMessageFactory(
+            self.amq_configuration.amq_adapter.service_name
+        )
+        self.backpressure_thread: Optional[Thread] = None
+
+    def run(self):
+        # self.init(loop) will initialize RabbitMQ connection
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_until_complete(self.init(self.loop, self.service_adapter))
+
+        logging.info("***********************************************")
+        logging.info("******  AMQ   Service  Init  Complete *********")
+        logging.info("***********************************************")
+        self.loop.run_forever()
+
+    async def init(self, loop, service_adapter):
+        _reply_to_exchange: AbstractRobustExchange = await self.rabbit_mq_client.init(
+            loop
+        )
+        backpressure_handler = BackpressureHandler(
+            channel=self.rabbit_mq_client.channel,
+            loop=loop,
+            config=self.amq_configuration,
+        )
+        self.amq_data_message_handler = DataMessageHandler(
+            service_adapter,
+            self.tracer,
+            self.data_message_factory,
+            self.service_message_factory,
+            _reply_to_exchange,
+            self.rabbit_mq_client,
+            self.amq_configuration,
+            loop=loop,
+            backpressure_handler=backpressure_handler,
+        )
+        await self.register_routes()
+        await self.rabbit_mq_client.request_routes()
+        self.backpressure_thread = backpressure_handler.start_backpressure_monitor()
 
     def cleanup(self):
-        logging.info(" [DBG] AMQService.cleanup() rmqc=%s", self.rabbit_mq_consumer)
-        if self.rabbit_mq_consumer:
-            self.rabbit_mq_consumer.cleanup()
+        logging_info("AMQService.cleanup() rmqc=%s", self.rabbit_mq_client)
+        if self.rabbit_mq_client:
+            self.rabbit_mq_client.cleanup()
 
-    def reinitialize_if_disconnected(self):
-        if not self.rabbit_mq_consumer.channel or not self.rabbit_mq_consumer.channel.is_open:
-            self.init()
+    async def reinitialize_if_disconnected(self):
+        if not self.rabbit_mq_client.channel or self.rabbit_mq_client.channel.is_closed:
+            self.rabbit_mq_client = RabbitMQClient(self.amq_configuration)
+            await self.rabbit_mq_client.init(loop=self.loop)
+            await self.register_routes()
+            await self.rabbit_mq_client.request_routes()
 
-    def setup_opentelemetry(self):
-        trace.set_tracer_provider(TracerProvider())
-        trace.get_tracer_provider().add_span_processor(
-            SimpleSpanProcessor(ConsoleSpanExporter())
-        )
-        set_global_textmap(TraceContextTextMapPropagator())
-        self.tracer = trace.get_tracer("com.cleverthis")
-        logging.info("OTEL %s %s %s", trace.get_logger_bridge(), trace.get_meter_provider(), trace.get_tracer_provider())
-        logging.info("OTEL TRACER %s", trace.get_propagators())
-
-    def register_routes(self):
-        route_mapping = self.amq_configuration.amq_adapter().route_mapping()
+    async def register_routes(self) -> AbstractRobustExchange | None:
+        route_mapping = self.amq_configuration.amq_adapter.route_mapping
         if AMQRouteFactory.validate(route_mapping):
             try:
-                self.rabbit_mq_consumer.register_routes()
-                self.rabbit_mq_consumer.register_deliver_callback(
-                    partial(self.amq_service_message_handler.on_delivery_callback_http, self.rabbit_mq_consumer.channel)
+                await self.rabbit_mq_client.register_inbound_routes(
+                    route_mapping,
+                    self.amq_data_message_handler.inbound_data_message_callback,
                 )
-                self.rabbit_mq_consumer.register_rpc_handler(
-                    self.amq_service_message_handler.on_reply_consumer(self.rabbit_mq_consumer.channel)
+                return await self.rabbit_mq_client.register_data_reply_callback(
+                    self.amq_data_message_handler.reply_received_callback
                 )
             except Exception as e:
                 raise RuntimeError(e)
         elif route_mapping:
-            logging.info(" [W] route [%s] failed validation, no routes were registered.", route_mapping)
-
-    def register_rpc_handler(self):
-        try:
-            self.rabbit_mq_consumer.register_rpc_handler(
-                self.amq_service_message_handler.on_reply_consumer(self.rabbit_mq_consumer.get_channel())
+            logging_warning(
+                "route [%s] failed validation, no routes were registered.",
+                route_mapping,
             )
-        except Exception as err:
-            raise
+        return None
+
+    def send_message(self, message: DataMessage, future: Future):
+        self.amq_data_message_handler.outstanding[str(message.id)] = future
+
+    def remove_outstanding_future(self, message_id: str):
+        self.amq_data_message_handler.outstanding.pop(message_id, None)

amqp/service/delete_queues.py

@@ -0,0 +1,48 @@
+import requests
+
+
+def delete_queues_with_prefix_http(
+    host="localhost",
+    port=15672,
+    username="springuser",
+    password="TheCleverWho",
+    prefix="cm-",
+):
+    """
+    CLI utility. Delete queues using RabbitMQ HTTP API.
+    """
+    base_url = f"http://{host}:{port}/api/queues/%2F"
+
+    try:
+        response = requests.get(base_url, auth=(username, password))
+        response.raise_for_status()
+
+        queues = [q["name"] for q in response.json() if q["name"].startswith(prefix)]
+
+        if not queues:
+            print(f"No queues found with prefix '{prefix}'")
+            return
+
+        print(f"Found {len(queues)} queues with prefix '{prefix}':")
+        for queue in queues:
+            print(f" - {queue}")
+
+        for queue in queues:
+            try:
+                delete_url = f"{base_url}/{queue}"
+                response = requests.delete(delete_url, auth=(username, password))
+                response.raise_for_status()
+                print(f"Successfully deleted queue: {queue}")
+            except Exception as e:
+                print(f"Failed to delete queue {queue}: {str(e)}")
+
+        print("Queue deletion process completed.")
+
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")
+
+
+if __name__ == "__main__":
+    delete_queues_with_prefix_http()
+    delete_queues_with_prefix_http(prefix="amq.gen--")
+    delete_queues_with_prefix_http(prefix="amq_")

amqp/service/logging.conf

@@ -0,0 +1,34 @@
+[loggers]
+keys=root,amqp_adapter
+
+[handlers]
+keys=consoleHandler,fileHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_amqp_adapter]
+level=DEBUG
+handlers=consoleHandler,fileHandler
+qualname=amqp_adapter
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=FileHandler
+level=DEBUG
+formatter=simpleFormatter
+args=('AMQ_PyAdapter.log',)
+
+[formatter_simpleFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+datefmt=%Y-%m-%d %H:%M:%S

amqp/service/service_message_handler.py

@@ -1,143 +0,0 @@
-import logging
-import time
-
-from typing import Callable, Dict
-from concurrent.futures import Future
-from opentelemetry import trace
-from opentelemetry.propagate import get_global_textmap, set_global_textmap
-from opentelemetry.propagators.tracecontext import TraceContextTextMapPropagator
-from opentelemetry.trace import Span, Tracer, TraceState
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-from amqp.model.model import AMQResponse
-from amqp.router.serializer import map_as_string
-
-
-class AMQServiceMessageHandler:
-    def __init__(self, http_adapter, tracer):
-        self.http_adapter = http_adapter
-        self.tracer = tracer
-        self.outstanding = {}
-
-    def on_delivery_callback_http(self, channel, consumerTag, delivery):
-        start = time.time()
-        deliveryTag = delivery.envelope.delivery_tag
-        debug = f" [DBG] ######[{deliveryTag}] (AMQ MESSAGE), env: {delivery.envelope}, props: {delivery.properties}"
-        logging.info(debug)
-
-        amq_message = AMQMessageFactory.from_bytes(delivery.body)
-        logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, traceInfo=%s",
-                    deliveryTag, consumerTag, amq_message.id, map_as_string(amq_message.trace_info))
-
-        propagator = TraceContextTextMapPropagator.get_instance()
-        context = propagator.extract(trace.get_current_context(), amq_message.trace_info    , self.get_text_getter())
-        parent_span = Span.from_context(context)
-        context = context.with_span(parent_span)
-
-        with context.activate():
-            logging.info(" [*] CTX=%s", context)
-            current_span = self.tracer.start_span("AMQ-Adapter.Processing.Request", parent=parent_span)
-            context_with_span = trace.set_span_in_context(current_span, context)
-            logging.info(" [*] CTX2=%s", context_with_span)
-            propagator.inject(context_with_span, amq_message.trace_info, self.set_text)
-            logging.info(" [***] NEW TI=%s CTX=%s SCOPE=%s",
-                        map_as_string(amq_message.trace_info), context_with_span, context)
-
-            if amq_message.magic == AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST:
-                future = self.http_adapter.handle(amq_message)
-                future.add_done_callback(
-                    lambda f: self.on_http_response(delivery, deliveryTag, channel, f.result())
-                )
-            else:
-                logging.error(" [E] Unknown MAGIC value: [%s], don't know how to interpret the message",
-                              amq_message.magic())
-
-            duration = time.time() - start
-            if "___NACK___" in str(amq_message.body()):
-                logging.info(" [x] ######[%s] Done, single NACK: %sms.", deliveryTag, duration)
-                channel.basic_nack(deliveryTag, False, False)
-            else:
-                logging.info(" [x] ######[%s] Done, single ACK: %sms.", deliveryTag, duration)
-                channel.basic_ack(deliveryTag, False)
-
-    def on_http_response(self, delivery, deliveryTag, channel, http_response):
-        response = AMQResponse(
-            delivery.properties.correlation_id,
-            http_response.status_code,
-            http_response.headers.get("Content-Type"),
-            http_response.body,
-            http_response.status_message,
-            "",
-            self.collect_trace_info(trace.get_current_context())
-        )
-        try:
-            self.send_reply(delivery.properties.reply_to, deliveryTag, channel, response)
-        except Exception as e:
-            raise RuntimeError(e)
-
-
-    class AMQServiceMessageHandler:
-        def __init__(self, http_adapter, tracer):
-            self.http_adapter = http_adapter
-            self.tracer = tracer
-            self.outstanding: Dict[str, 'ResponseHolder'] = {}
-
-        def collect_trace_info(self, context):
-            map = {}
-            TraceContextTextMapPropagator.get_instance().inject(
-                context if context else trace.get_current_context(),
-                map, self.set_text
-            )
-            return map
-
-        def send_reply(self, reply_to, delivery_tag, channel, response):
-            if reply_to:
-                logging.info(" [*] ######[%s] AMQ Message Reply, To=%s, msgId=%s",
-                             delivery_tag, reply_to, response.id())
-                reply_props = pika.BasicProperties(
-                    correlation_id=response.id(),
-                    content_type=response.content_type()
-                )
-                channel.basic_publish(
-                    RouterBase.AMQ_REPLY_TO_EXCHANGE,
-                    reply_to,
-                    reply_props,
-                    AMQResponseFactory.serialize(response)
-                )
-
-        def on_reply_consumer(self, channel):
-            def handle_delivery(consumerTag, envelope, properties, body):
-                reply_id = properties.correlation_id
-                content_type = properties.content_type
-                delivery_tag = envelope.delivery_tag
-                debug = f" [DBG] ######[{delivery_tag}]     (RPC REPLY) env: {envelope}, props: {properties}, BODY: {body.decode()}"
-                logging.info(debug)
-                print(f"OUTSTANDING RECV: {self.outstanding}, size={len(self.outstanding)}")
-
-                with self.outstanding.lock:
-                    reply = self.outstanding.pop(reply_id, None)
-                    if reply is None:
-                        logging.warn(" [W] ######[%s] IGNORING REPLY. No outstanding request for ID %s",
-                                     delivery_tag, reply_id)
-                    else:
-                        result = AMQResponseFactory.from_bytes(body)
-                        reply.tracing_span().end()
-                        reply.response().set_result(Response.ok(result.body(), content_type).build())
-
-                channel.basic_ack(delivery_tag, False)
-
-            return handle_delivery
-
-        def register_response_holder(self, id, response_holder):
-            self.outstanding[id] = response_holder
-            logging.info(" [*] AMQMessage Sent, OUTSTANDING response holder registered: %s, size=%s",
-                         self.outstanding, len(self.outstanding))
-
-        def get_default_trace_state(self):
-            return TraceState(entries={})
-
-        def get_text_getter(self):
-            return lambda map, key: map.get(key)
-
-        def set_text(self, carrier, key, value):
-            carrier[key] = value

amqp/service/tracing.py

@@ -1,7 +1,6 @@
 from opentelemetry import metrics, trace
-from opentelemetry.instrumentation.flask import FlaskInstrumentor
 from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace import TracerProvider, Tracer
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 
@@ -11,22 +10,22 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource
 
 import os
 
-def initialize_trace(flask_app, name):
+
+def initialize_trace(app=None, name=None) -> Tracer:
     # Resource can be required for some backends, e.g. Jaeger or Prometheus
     # If resource wouldn't be set - traces wouldn't appear in Jaeger.
     # The 'name' value is the name shown in Jaeger.
-    resource = Resource(attributes={
-        SERVICE_NAME: name if name else os.environ.get('APPLICATION_NAME', 'Python App')
-    })
+    resource = Resource(
+        attributes={
+            SERVICE_NAME: (
+                name if name else os.environ.get("APPLICATION_NAME", "Python App")
+            )
+        }
+    )
     #
     # 1. Tracing (Jaeger) configuration
     # ----------------------------------
     #
-    # Initialize the tracer provider. This is abstract API call.
-    # The actual implementation is given by the included dependency
-    # The opentelemetry default is the OTLP trace provider.
-    trace.set_tracer_provider(TracerProvider(resource=resource))
-    #
     # Configure the OTLP exporter (or HTTP exporter the same way)
     # the recommended configuration is via environment variables. The mandatory ones:
     #       OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
@@ -34,14 +33,27 @@ def initialize_trace(flask_app, name):
     # for more see here: https://opentelemetry-python.readthedocs.io/en/latest/exporter/otlp/otlp.html
     # alternatively these values can be provided as input values to OTLPSpanExporter() constructor.
     #
-    otlp_exporter = OTLPSpanExporter()
+    # otlp_exporter = OTLPSpanExporter()
+    # or provide endpoint explicitly, as
+    otlp_exporter = OTLPSpanExporter(endpoint="http://jaeger:4317")
     #
+    # Initialize the tracer provider. This is abstract API call.
+    # The actual implementation is given by the included dependency
+    # The opentelemetry default is the OTLP trace provider.
+    provider = TracerProvider(resource=resource)
     # Add the exporters to the batch span processor
     span_processor = BatchSpanProcessor(otlp_exporter)
-    trace.get_tracer_provider().add_span_processor(span_processor)
+    provider.add_span_processor(span_processor)
+    # Sets the global default tracer provider
+    trace.set_tracer_provider(provider)
+    #
+
+    # logging.info(' * Instrumenting application..... ')
+    # if Flask, then use:
+    # FlaskInstrumentor().instrument_app(flask_app)
+    # if FastAPI, then use
+    # FastAPIInstrumentor.instrument_app(app)
 
-    print(' * Instrumenting Flask application..... ')
-    FlaskInstrumentor().instrument_app(flask_app)
     #
     # Python Flask is now automatically instrumented to export details of each REST endpoint call.
     # For Django based app, use 'opentelemetry-instrumentation-django' import instead. see here:
@@ -64,8 +76,8 @@ def initialize_trace(flask_app, name):
     #         kind=trace.SpanKind.SERVER,
     #         attributes=collect_request_attributes(request.environ),
     #     ):
-    #         # do the work here, like print something and return response
-    #         print(request.args.get("param"))
+    #         # do the work here, like logging.info something and return response
+    #         logging.info(request.args.get("param"))
     #         return "served"
     #
     # Note:
@@ -74,7 +86,6 @@ def initialize_trace(flask_app, name):
     #    of that parent span, indicating visually the progression of requests via different services
     #    for as long as the current span is provided in the traceparent header to next service.
 
-
     # 2. Metrics (Prometheus) configuration
     # -------------------------------------
     #
@@ -87,3 +98,5 @@ def initialize_trace(flask_app, name):
     reader = PrometheusMetricReader()
     provider = MeterProvider(resource=resource, metric_readers=[reader])
     metrics.set_meter_provider(provider)
+
+    return trace.get_tracer(name)

cleverswarm/clever_swarm_adapter.py

@@ -0,0 +1,302 @@
+"""
+This module provides the CleverSwarm AMQP Adapter for handling API requests via AMQP.
+The code IS MEANT to be part of CleverSwarm codebase, and has direct dependencies on the CleverSwarm codebase.
+It is NOT MEANT to be used as a standalone module. It WILL NOT run outside the CleverSwarm endpoints context.
+"""
+
+import importlib
+import pathlib
+import re
+from collections import defaultdict
+from threading import Thread
+from typing import Optional, List, Dict
+from aiohttp import ClientSession
+from fastapi.routing import APIRoute
+from fastapi.security import OAuth2PasswordRequestForm
+from starlette.datastructures import UploadFile, Headers
+
+# CleverSwarm specific imports
+import core.deps
+from rest.v0 import api
+from schemas.user_schema import UserSchema
+
+from amqp.adapter.data_parser import AMQDataParser
+
+# AMQP specific imports
+from amqp.adapter.data_response_factory import AMQResponseFactory
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, AMQMessage
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import AMQResponse
+from amqp.service.amq_service import AMQService
+
+
+# Suffix for the method name with serialization wrapper, prefer use this one over the orig endpoint
+PREFERRED_SUFFIX = "_json"
+
+
+def clone_and_adapt_route(original_route: APIRoute) -> APIRoute:
+    """
+    Creates a clone of an APIRoute. It checks if there's preferred endpoint with json serialization.
+    It also adjusts the path regex to ensure it matches the path in different API versions.
+    """
+
+    # Check if the route has a preferred endpoint with json serialization
+    _preferred_endpoint = get_prefered_endpoint(
+        getattr(original_route.endpoint, "__module__"),
+        original_route.endpoint.__name__ + PREFERRED_SUFFIX,
+    )
+    # Need to adapt also the path regex to ensure it matches the path in different API versions
+    # Ensures the leading '^' is followed by any match and there's no '/' before final '$'
+    _new_regex_string = (
+        "^.*" + original_route.path_regex.pattern.lstrip("^").rstrip("/$") + "$"
+    )
+    _new_route: APIRoute = APIRoute(
+        path=original_route.path,
+        endpoint=(
+            _preferred_endpoint if _preferred_endpoint else original_route.endpoint
+        ),
+        response_model=original_route.response_model,
+        status_code=original_route.status_code,
+        tags=original_route.tags.copy() if original_route.tags else None,
+        dependencies=(
+            original_route.dependencies.copy() if original_route.dependencies else None
+        ),
+        summary=original_route.summary,
+        description=original_route.description,
+        response_description=original_route.response_description,
+        responses=original_route.responses.copy() if original_route.responses else None,
+        deprecated=original_route.deprecated,
+        name=original_route.name,
+        methods=original_route.methods.copy() if original_route.methods else None,
+        operation_id=original_route.operation_id,
+        response_model_include=original_route.response_model_include,
+        response_model_exclude=original_route.response_model_exclude,
+        response_model_by_alias=original_route.response_model_by_alias,
+        response_model_exclude_unset=original_route.response_model_exclude_unset,
+        response_model_exclude_defaults=original_route.response_model_exclude_defaults,
+        response_model_exclude_none=original_route.response_model_exclude_none,
+        include_in_schema=original_route.include_in_schema,
+        response_class=original_route.response_class,
+        dependency_overrides_provider=original_route.dependency_overrides_provider,
+        callbacks=original_route.callbacks.copy() if original_route.callbacks else None,
+        openapi_extra=(
+            original_route.openapi_extra.copy()
+            if original_route.openapi_extra
+            else None
+        ),
+        generate_unique_id_function=original_route.generate_unique_id_function,
+    )
+    _new_route.path_regex = re.compile(_new_regex_string)
+    return _new_route
+
+
+def get_prefered_endpoint(module_name: str, function_name: str) -> Optional[callable]:
+    """Get the function if it exists in the module, else return None."""
+    try:
+        _module = importlib.import_module(module_name)
+        _prefered_endpoint = getattr(_module, function_name)
+        return _prefered_endpoint if callable(_prefered_endpoint) else None
+    except Exception:
+        return None
+
+
+def group_routes_by_method(api_routes: List[APIRoute]) -> Dict[str, List[APIRoute]]:
+    """
+    Groups API routes by their HTTP method. This is important to ensure that the URLPath is matched
+    correctly, as different methods may share the same URLPath (e.g. GET, POST PUT, DELETE).
+    """
+    _method_to_routes = defaultdict(
+        list
+    )  # Automatically initializes new lists for new keys
+    for _route in api_routes:
+        if (
+            _route.methods
+        ):  # Ensure methods exist (should always be true for valid routes)
+            for method in _route.methods:
+                _method_to_routes[method].append(clone_and_adapt_route(_route))
+
+    return dict(_method_to_routes)
+
+
+class CleverSwarmAmqpAdapter(CleverThisServiceAdapter):
+    """
+    CleverSwarm AMQP Adapter for CleverSwarm API. Implements the interface to CleverSwarm API.
+    """
+
+    def __init__(
+        self, amq_configuration: AMQConfiguration, session: ClientSession = None
+    ):
+        super().__init__(amq_configuration, session)
+        self.routes = group_routes_by_method(api_routes=api.api_router.routes)
+        _amq_service: AMQService = AMQService(amq_configuration, self)
+        self.thread = Thread(target=_amq_service.run)
+        self.thread.start()
+
+    async def get_current_user(self, token: str) -> UserSchema:
+        """
+        Overrides the default get_current_user method to use the token from the AMQP message.
+        :param token: JWT token from the AMQP message
+        :return: UserSchema object
+        """
+        return await core.deps.get_current_user(token)
+
+    # ==================================================================================================
+    # =====================  C l e v e r S w a r m   A P I   m e t h o d s  ============================
+    # ==================================================================================================
+
+    async def get(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the GET requests for the CleverSwarm API.
+        :param message: message from the AMQP that contains the GET request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        return await self.handle_no_body_payload(message, auth_user, "GET")
+
+    async def put(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the PUT requests for the CleverSwarm API.
+        :param message: message from the AMQP that contains the PUT request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        return await self.handle_body_payload(message, auth_user, "PUT")
+
+    async def post(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the POST requests for the CleverSwarm API.
+        :param message: message from the AMQP that contains the POST request, including body and headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        return await self.handle_body_payload(message, auth_user, "POST")
+
+    async def head(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the HEAD requests for the CleverSwarm API.
+        """
+        return await self.handle_no_body_payload(message, auth_user, "HEAD")
+
+    async def delete(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the DELETE requests for the CleverSwarm API.
+        """
+        return await self.handle_no_body_payload(message, auth_user, "DELETE")
+
+    async def options(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the OPTIONS requests for the CleverSwarm API.
+        """
+        return await self.handle_no_body_payload(message, auth_user, "OPTIONS")
+
+    async def patch(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the PATCH requests for the CleverSwarm API.
+        """
+        return await self.handle_no_body_payload(message, auth_user, "PATCH")
+
+    async def handle_no_body_payload(
+        self, message: AMQMessage, auth_user: UserSchema, method: str
+    ) -> AMQResponse:
+        """
+        Handles the requests that do not have a body payload.
+        :param message: message from the AMQP that contains the request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        _routes = self.routes.get(method, [])
+        _path, _args = AMQDataParser.parse_get_url(message.path)
+        for _route in _routes:
+            _match = _route.path_regex.match(_path)
+            if _match:
+                _args.update(_match.groupdict())
+                _args["authenticated_user"] = auth_user
+                # If the path matches, call the corresponding function
+                return await CleverThisServiceAdapter.call_cleverthis_api(
+                    message,
+                    _args,
+                    api_method=_route.endpoint,
+                    success_code=_route.status_code,
+                    loop=self.loop,
+                )
+
+        return AMQResponseFactory.of(
+            message.id,
+            404,
+            "text/plain",
+            str(f"Destination location [{message.path}] does not exists").encode(
+                "utf-8"
+            ),
+            message.trace_info,
+        )
+
+    async def handle_body_payload(
+        self, message: AMQMessage, auth_user: UserSchema, method: str
+    ) -> AMQResponse:
+        """
+        Handles the requests that have a body payload, including Multipart data.
+        :param message: message from the AMQP that contains the request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :param method: HTTP method (POST, PUT, etc.)
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        _form_data = AMQDataParser.parse_request_body(message)
+        _routes = self.routes.get(method, [])
+        _path, _query_args = AMQDataParser.parse_get_url(message.path)
+        for _route in _routes:
+            _match = _route.path_regex.match(_path)
+            if _match:
+                # If the path matches, call the corresponding function
+                _form_data.update(_match.groupdict())
+                _form_data.update(_query_args)
+                for _body_param in _route.dependant.body_params:
+                    if _body_param.name in _form_data:
+                        _file_data = _form_data[_body_param.name]
+                        actual_filepath = message.extra_data[_body_param.name]
+                        _form_data[_body_param.name] = (
+                            _body_param.type_.__name__ == "UploadFile"
+                            and UploadFile(
+                                file=pathlib.Path(actual_filepath).open("rb"),
+                                size=_file_data["size"],
+                                filename=_file_data["filename"],
+                                headers=Headers(
+                                    {"Content-Type": _file_data["mediaType"]}
+                                ),
+                            )
+                            or _form_data[_body_param.name]
+                        )
+
+                _verified_args = {}
+                _required_args = getattr(_route.endpoint, "__annotations__", {})
+                for arg in _required_args:
+                    if arg in _form_data:
+                        _verified_args[arg] = _form_data[arg]
+                    else:
+                        # needed for /login  input
+                        if arg == "authenticated_user":
+                            _verified_args["authenticated_user"] = auth_user
+                        if _required_args[arg].__name__ == "OAuth2PasswordRequestForm":
+                            try:
+                                _verified_args[arg] = OAuth2PasswordRequestForm(
+                                    **_form_data
+                                )
+                            except Exception as e:
+                                print(f"Error: {e}")
+
+                return await CleverThisServiceAdapter.call_cleverthis_api(
+                    message,
+                    _verified_args,
+                    api_method=_route.endpoint,
+                    success_code=_route.status_code if _route.status_code else 200,
+                    loop=self.loop,
+                )
+
+        return AMQResponseFactory.of(
+            message.id,
+            404,
+            "text/plain",
+            str(f"Destination location [{message.path}] does not exists").encode(
+                "utf-8"
+            ),
+            message.trace_info,
+        )

cleverswarm/main.py

@@ -0,0 +1,8 @@
+from amqp.config.amq_configuration import AMQConfiguration
+
+from clever_swarm_adapter import CleverSwarmAmqpAdapter
+
+if __name__ == "__main__":
+    adapter = CleverSwarmAmqpAdapter(AMQConfiguration("./application.properties.local"))
+    print("Press ^C to exit...")
+    adapter.thread.join()

compose.yaml

@@ -0,0 +1,244 @@
+name: clever_micro
+services:
+
+  rabbitmq: # copied from cleverdata backend branch feat/#190
+    image: rabbitmq:3-management
+    hostname: "my-rabbit"
+    environment:
+      RABBITMQ_DEFAULT_USER: springuser
+      RABBITMQ_DEFAULT_PASS: TheCleverWho
+    volumes:
+      - "./container-data/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins"
+    #      - "./container-data/mq-data:/var/lib/rabbitmq"
+    ports:
+      - "5672:5672"
+      - "8888:15672"
+    healthcheck:
+      test: ["CMD-SHELL", "rabbitmq-diagnostics -q ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  traefik:
+    image: "traefik:v3.2"
+    container_name: "traefik"
+    command:
+      - "--api.insecure=true"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entryPoints.web.address=:8085"
+    labels:
+      - "traefik.enable=true"
+    ports:
+      - "8085:8085"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+
+  jaeger:
+    image: "jaegertracing/all-in-one:latest"
+    ports:
+      - "16686:16686"  # Jaeger UI
+      - "14268:14268"  # Jaeger collector Accepts jaeger.thrift spans directly over HTTP
+      - "14250:14250"  # Jaeger gRPC Accepts model.proto spans over gRPC
+      - "6831:6831/udp"  # Jaeger agent Accepts jaeger.thrift spans (Thrift compact)
+      - "6832:6832/udp"  # Jaeger agent Accepts jaeger.thrift spans (Thrift binary)
+      - "5778:5778"    # Jaeger agent admin + configuration
+      - "4317:4317"    # OpenTelemetry Protocol (OTLP) gRPC receiver
+      - "4318:4318"    # OpenTelemetry Protocol (OTLP) HTTP/protobuf receiver
+      - "14269:14269"  # Jaeger health check
+      - "9411:9411"    # Zipkin compatibility
+    environment:
+      COLLECTOR_ZIPKIN_HTTP_PORT: 9411
+      COLLECTOR_OTLP_ENABLED: true
+
+  amqp-router:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    #  dockerfile: Dockerfile.amqp-adapter
+    #image: "amq-python:latest"
+    image: "amqp_router:latest"
+    ports:
+      - "${AMQ_ADAPTER_PORT:-8081}:8080"
+    environment:
+      cm.amq-adapter.service-name: "amq-router"
+      cm.amq-adapter.is-router: "true"
+      cm.amq-adapter.generator-id: 164
+      cm.amq-adapter.route-mapping: "null"
+
+      cm.dispatch.use-dlq: true
+      cm.dispatch.use-confirms: false
+      cm.dispatch.amq-host: rabbitmq
+      cm.dispatch.amq-port: 5672
+      cm.dispatch.amq-port-tls: 5671
+
+      cm.backpressure.threshold: 20
+      cm.backpressure.time-window: 3000
+      cm.backpressure.management-service-url: "management-service:8080"
+      quarkus.otel.exporter.otlp.endpoint: "http://jaeger:4317"
+      quarkus.application.name: API-Gateway-AMQ-Adapter
+      quarkus.otel.service.name: API-Gateway-AMQ-Adapter
+    env_file:
+      - .env_secrets
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.router-app.rule=PathPrefix(`/`)"  # default route
+      - "traefik.http.routers.router-app.entrypoints=web"
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "curl http://localhost:8080/amq-adapter-healthcheck"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  #
+  # DLQ is special case of AMQ adapter that will handle rejected messages
+  #
+  dead-letter-queue-handler:
+    image: "amqp_router:latest"
+    ports:
+      - "${DLQ_ADAPTER_PORT:-8089}:8080"
+    environment:
+
+      cm.amq-adapter.service-name: "amq-dlq"
+      cm.amq-adapter.is-router: "false"
+      cm.amq-adapter.generator-id: 165
+      cm.amq-adapter.route-mapping: "amq-dlq:DLX:DeadLetterQueue:#:0:0"
+
+      cm.dispatch.use-dlq: false
+      cm.dispatch.use-confirms: false
+      cm.dispatch.amq-host: rabbitmq
+      cm.dispatch.amq-port: 5672
+      cm.dispatch.amq-port-tls: 5671
+
+      cm.backpressure.threshold: 20
+      cm.backpressure.time-window: 3000
+      cm.backpressure.management-service-url: "management-service:8080"
+      quarkus.application.name: AMQ-DeadLetterQueue
+      quarkus.otel.exporter.otlp.endpoint: "http://jaeger:4317"
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+    env_file:
+      - .env_secrets
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+  #
+  # test applications
+  #
+  amq-clever-book:
+    image: "amqp_router:latest"
+    ports:
+      - "16000:8080"
+    environment:
+
+      cm.amq-adapter.service-name: "amq-cleverbook-adapter"
+      cm.amq-adapter.generator-id: 16
+      cm.amq-adapter.route-mapping: "amq-clever-book::amq-clever-book:clever.book.localhost.#:5:0"
+
+      cm.dispatch.use-dlq: true
+      cm.dispatch.use-confirms: false
+      cm.dispatch.amq-host: rabbitmq
+      cm.dispatch.amq-port: 5672
+      cm.dispatch.amq-port-tls: 5671
+      cm.dispatch.service-host: clever-book
+      cm.dispatch.service-port: 8080
+
+      cm.backpressure.threshold: 20
+      cm.backpressure.time-window: 3000
+      cm.backpressure.management-service-url: "management-service:8080"
+      quarkus.otel.service.name: CleverBook-AMQ-Adapter
+      quarkus.application.name: CleverBook-AMQ-Adapter
+      quarkus.otel.exporter.otlp.endpoint: "http://jaeger:4317"
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+    env_file:
+      - .env_secrets
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+
+  #
+  # Dummy app allowing POST and GET operations over REST API, emulating a real SpringBoot service
+  #
+  clever-book:
+    build:
+      context: clever-book/
+      dockerfile: Dockerfile
+    image: "book-management:latest"
+    ports:
+      - "16002:8080"
+    restart: on-failure
+    environment:
+      SPRING_PROFILES_ACTIVE: prod
+      SPRING_APPLICATION_NAME: CleverBook
+      spring.application.name: CleverBook
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+      otel.exporter.otlp.endpoint: "http://jaeger:4317"
+    env_file:
+      - .env_secrets
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+
+  #
+  # a black box 3rd party app provisioned as-is
+  #
+  black-box-3rd-party:
+    image: "traefik/whoami"
+    container_name: "blackbox-third-party"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.whoami.rule=Host(`whoami.localhost`)"
+      - "traefik.http.routers.whoami.entrypoints=web"
+
+  #
+  # test applications for python
+  #
+  amq-clever-pybook:
+    image: "amq-python:latest"
+    # image: "amqp_router:latest"
+    ports:
+      - "16020:8080"
+    environment:
+
+      cm.amq-adapter.service-name: "amq-cleverpybook-adapter"
+      cm.amq-adapter.generator-id: 211
+      cm.amq-adapter.route-mapping: "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+
+      cm.dispatch.use-dlq: true
+      cm.dispatch.use-confirms: false
+      cm.dispatch.amq-host: rabbitmq
+      cm.dispatch.amq-port: 5672
+      cm.dispatch.amq-port-tls: 5671
+      cm.dispatch.service-host: clever-pybook
+      cm.dispatch.service-port: 5000
+
+      cm.backpressure.threshold: 20
+      cm.backpressure.time-window: 3000
+      cm.backpressure.management-service-url: "management-service:8080"
+      quarkus.otel.service.name: CleverPyBook-AMQ-Adapter
+      quarkus.application.name: CleverPyBook-AMQ-Adapter
+      quarkus.otel.exporter.otlp.endpoint: "http://jaeger:4317"
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+    env_file:
+      - .env_secrets
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+
+  #
+  # Dummy app allowing POST and GET operations over REST API, emulating a real SpringBoot service
+  #
+  clever-pybook:
+    image: "clever-pybook:latest"
+    ports:
+      - "16012:5000"
+    restart: on-failure
+    environment:
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+      OTEL_EXPORTER_OTLP_INSECURE: "true"
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+

container-data/alertmanager/alertmanager.yml

@@ -0,0 +1,16 @@
+route:
+  receiver: 'demo.web.hook'
+  group_by: ['path', 'queue']
+  group_wait: 3s
+  group_interval: 1s
+  repeat_interval: 1m
+receivers:
+  - name: 'demo.web.hook'
+    webhook_configs:
+      - url: 'http://10.233.1.4:8080/alert'
+inhibit_rules:
+  # override warnings if we have critical
+  - source_match:
+      severity: 'critical'
+    target_match:
+      severity: 'warning'

container-data/grafana/dashboard-export/Application-1730708374720.json

@@ -0,0 +1,271 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "11.3.0"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.3.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "rabbitmq_queue_messages_ready{queue=\"demo.getRandom\"}",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false,
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          }
+        }
+      ],
+      "title": "RabbitMQ Queue getRandom",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.3.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "delta(application_demo_http_req_duration_second_seconds_count[$__interval])",
+          "fullMetaSearch": false,
+          "includeNullMetadata": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "/number req",
+      "type": "timeseries"
+    }
+  ],
+  "schemaVersion": 40,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Application",
+  "uid": "ce2wmjrfkm7lsf",
+  "version": 5,
+  "weekStart": ""
+}

container-data/kibana/config/kibana.yml

@@ -0,0 +1,9 @@
+server.host: 0.0.0.0
+elasticsearch.hosts: [ http://elasticsearch:9200 ]
+
+monitoring.ui.container.elasticsearch.enabled: true
+monitoring.ui.container.logstash.enabled: true
+
+# X-Pack security credentials
+elasticsearch.username: kibana_system
+elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}

container-data/logstash/pipeline/logstash.conf

@@ -0,0 +1,15 @@
+input {
+	beats {
+		port => 5044
+	}
+}
+
+## Add your filters / logstash plugins configuration here
+
+output {
+	elasticsearch {
+		hosts => "elasticsearch:9200"
+		user => "logstash_internal"
+		password => "${LOGSTASH_INTERNAL_PASSWORD}"
+	}
+}

container-data/prometheus/alert-rules.yml

@@ -0,0 +1,21 @@
+groups:
+  - name: demo_application
+    rules:
+      - alert: application_http_duration_alert
+        expr: rate(application_demo_http_req_duration_second_seconds_bucket{le="+Inf"}[5s]) >= 100
+        for: 5s
+        labels:
+          severity: critical
+        annotations:
+          summary: "Application process http req take too long"
+          description: "Endpoint {{ $labels.path }} is taking too long"
+  - name: rabbitmq
+    rules:
+      - alert: rabbitmq_queue_message_ready_count_alert
+        expr: rabbitmq_queue_messages_ready >= 50
+        for: 3s
+        labels:
+          severity: critical
+        annotations:
+          summary: "RabbitMQ queue backpressure too high"
+          description: "Queue {{ $labels.queue }} is taking too long"

container-data/prometheus/prometheus.yml

@@ -0,0 +1,32 @@
+scrape_configs:
+  - job_name: RabbitMQ
+    scrape_interval: 2s
+    metrics_path: /metrics/per-object
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+    relabel_configs:
+      # Only keep containers that have a `prometheus-job=rabbitmq` label.
+      # also requires the port = 15962, that's the report port of RabbitMQ
+      - source_labels: [__meta_docker_container_label_prometheus_job, __meta_docker_port_private]
+        regex: rabbitmq;15692
+        action: keep
+
+  - job_name: SpringActuator
+    metrics_path: /actuator/prometheus
+    scrape_interval: 2s
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+    relabel_configs:
+      - source_labels: [__meta_docker_container_label_prometheus_job, __meta_docker_port_private]
+        regex: spring-actuator;8080
+        action: keep
+
+alerting:
+  alertmanagers:
+    - scheme: http
+      static_configs:
+        - targets:
+          - "alertmanager:9093"
+
+rule_files:
+  - "alert-rules.yml"

container-data/rabbitmq/enabled_plugins

@@ -0,0 +1 @@
+[rabbitmq_management,rabbitmq_prometheus].

container-data/traefik/traefik.yml

@@ -0,0 +1,20 @@
+api:
+  insecure: true
+  dashboard: true
+  debug: true
+  disableDashboardAd: true
+providers:
+  docker:
+    exposedbydefault: false
+log:
+  level: "DEBUG"
+
+entryPoints:
+  web:
+    address: ":8080"
+
+#  ampqs:
+#    address: ":3333"
+
+#  websecure:
+#    address: ":443"

pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]  # Specifies build tools to use
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "amq_adapter"
+version = "0.2.23-dev"
+description = "The Python implementation of the AMQ Adaptor for CleverMicro"
+readme = "README.md"
+authors = [
+    { name = "Stanislav Hejny", email = "stanislav.hejny@cleverthis.com" }
+]
+license = { text = "Apache License 2.0" }
+urls = { Homepage = "https://git.cleverthis.com/clevermicro/amq-adapter-python" }
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+requires-python = ">=3.11"
+dependencies = [
+    "aio-pika~=9.5",
+    "aiohttp~=3.11",
+    "aio_pika~=9.5",
+    "fastapi~=0.114",
+    "opentelemetry-api",
+    "opentelemetry-sdk",
+    "opentelemetry-exporter-otlp",
+    "opentelemetry-exporter-prometheus",
+    "opentelemetry-instrumentation-pika",
+    "python_multipart"
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["amqp*"]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "build",
+    "twine"
+]
+

requirements.txt

@@ -1,8 +0,0 @@
-aiohttp~=3.11.11
-pika~=1.3.2
-flask
-opentelemetry-api
-opentelemetry-sdk
-opentelemetry-exporter-otlp
-opentelemetry-exporter-prometheus
-opentelemetry-instrumentation-flask

tests/adapter/test_amq_route_factory.py

@@ -0,0 +1,30 @@
+from unittest import TestCase
+
+from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
+from amqp.model.model import AMQRoute
+
+
+class TestAMQRouteFactory(TestCase):
+    def test_from_string(self):
+        _f = AMQRouteFactory()
+        _r: AMQRoute = _f.from_string(
+            "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+        )
+        self.assertEqual("", _r.exchange)
+        self.assertEqual("amq-clever-pybook", _r.queue)
+        self.assertEqual(5, _r.timeout)
+        _r2: AMQRoute = _f.from_string(
+            "amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#:5"
+        )
+        self.assertEqual(NULL_ROUTE, _r2)
+
+    def test_validate(self):
+        _f = AMQRouteFactory()
+        self.assertTrue(
+            _f.validate(
+                "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+            )
+        )
+        self.assertFalse(
+            _f.validate("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#")
+        )

tests/adapter/test_cleverthis_service_adapter.py

@@ -0,0 +1,227 @@
+import unittest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from aiohttp import ClientResponse
+
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, AMQMessage
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import DataMessage, DataResponse
+
+
+class TestCleverThisServiceAdapter(unittest.TestCase):
+    def setUp(self):
+        # Mock AMQConfiguration
+        self.mock_amq_config = AMQConfiguration("./application.properties.local")
+
+        # Mock ClientSession
+        self.mock_session = AsyncMock()
+
+        # Create adapter instance
+        self.adapter = CleverThisServiceAdapter(
+            amq_configuration=self.mock_amq_config, session=self.mock_session
+        )
+
+    def test_init(self):
+        self.assertEqual(self.adapter.service_host, "localhost")
+        self.assertEqual(self.adapter.service_port, 8080)
+        self.assertFalse(self.adapter.require_authenticated_user)
+
+    def test_get_content_type(self):
+        test_cases = [
+            ({"Content-Type": ["application/json"]}, "application/json"),
+            ({"content-type": ["text/xml"]}, "text/xml"),
+            ({"CONTENT-TYPE": ["text/plain; charset=utf-8"]}, "text/plain"),
+            ({}, "application/json"),
+            ({"Other-Header": ["value"]}, "application/json"),
+        ]
+
+        for headers, expected in test_cases:
+            with self.subTest(headers=headers):
+                result = self.adapter.get_content_type(headers)
+                self.assertEqual(result, expected)
+
+    async def test_get_current_user(self):
+        # Base implementation should return None
+        result = await self.adapter.get_current_user("test_token")
+        self.assertIsNone(result)
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_unauthenticated(self, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers = {}
+        mock_message.path = "/some/path"
+        mock_message.method = "GET"
+        mock_message.id = "123"
+        mock_message.trace_info = {}
+
+        # Test when authentication is required but not provided
+        response = await self.adapter.on_message(mock_message)
+        self.assertEqual(response.response_code, 401)
+        mock_get_user.assert_not_called()
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    @patch.object(CleverThisServiceAdapter, "get")
+    async def test_on_message_authenticated(self, mock_get, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers = {"Authorization": ["Bearer valid_token"]}
+        mock_message.path = "/some/path"
+        mock_message.method = "GET"
+        mock_message.id = "123"
+        mock_message.trace_info = {}
+
+        mock_user = MagicMock()
+        mock_get_user.return_value = mock_user
+
+        mock_response = MagicMock(spec=DataResponse)
+        mock_get.return_value = mock_response
+
+        response = await self.adapter.on_message(mock_message)
+
+        self.assertEqual(response, mock_response)
+        mock_get_user.assert_called_once_with("valid_token")
+        mock_get.assert_called_once_with(mock_message, mock_user)
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_login_path(self, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers = {}
+        mock_message.path = "/login"
+        mock_message.method = "POST"
+        mock_message.id = "123"
+        mock_message.trace_info = {}
+
+        mock_response = MagicMock(spec=DataResponse)
+        with patch.object(
+            self.adapter, "post", return_value=mock_response
+        ) as mock_post:
+            response = await self.adapter.on_message(mock_message)
+            self.assertEqual(response, mock_response)
+            mock_get_user.assert_not_called()
+            mock_post.assert_called_once()
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_all_methods(self, mock_get_user):
+        mock_user = MagicMock()
+        mock_get_user.return_value = mock_user
+
+        mock_response = MagicMock(spec=DataResponse)
+
+        methods = ["GET", "PUT", "POST", "DELETE", "HEAD"]
+        for method in methods:
+            with self.subTest(method=method):
+                mock_message = MagicMock(spec=AMQMessage)
+                mock_message.headers = {"Authorization": ["Bearer valid_token"]}
+                mock_message.path = "/some/path"
+                mock_message.method = method
+                mock_message.id = "123"
+                mock_message.trace_info = {}
+
+                with patch.object(
+                    self.adapter, method.lower(), return_value=mock_response
+                ) as mock_method:
+                    response = await self.adapter.on_message(mock_message)
+                    self.assertEqual(response, mock_response)
+                    mock_method.assert_called_once()
+
+    async def test_get_with_session(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.headers = {"X-Test": "value"}
+        mock_message.trace_info = {"trace-id": "123"}
+        mock_message.id = "msg123"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 200
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "success"}')
+
+        self.mock_session.get.return_value = mock_response
+
+        response = await self.adapter.get(mock_message, None)
+
+        self.assertEqual(response.response_code, 200)
+        self.mock_session.get.assert_called_once_with(
+            "http://testhost:8080/api/test",
+            headers={"X-Test": "value", "trace-id": "123"},
+        )
+
+    async def test_get_without_session(self):
+        self.adapter.session = None
+
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.id = "msg123"
+        mock_message.trace_info = {}
+
+        response = await self.adapter.get(mock_message, None)
+
+        self.assertEqual(response.response_code, 404)
+
+    async def test_put_with_form_handling(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.headers = {}
+        mock_message.trace_info = {}
+        mock_message.id = "msg123"
+        mock_message.body = b"test=value"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 200
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "success"}')
+
+        self.mock_session.put.return_value = mock_response
+
+        with patch.object(
+            self.adapter,
+            "handle_possible_form",
+            wraps=self.adapter.handle_possible_form,
+        ) as mock_handler:
+            response = await self.adapter.put(mock_message, None)
+
+            self.assertEqual(response.response_code, 200)
+            mock_handler.assert_called_once()
+
+    async def test_post_with_form_handling(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.headers = {}
+        mock_message.trace_info = {}
+        mock_message.id = "msg123"
+        mock_message.body = b"test=value"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 201
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "created"}')
+
+        self.mock_session.post.return_value = mock_response
+
+        with patch.object(
+            self.adapter,
+            "handle_possible_form",
+            wraps=self.adapter.handle_possible_form,
+        ) as mock_handler:
+            response = await self.adapter.post(mock_message, None)
+
+            self.assertEqual(response.response_code, 201)
+            mock_handler.assert_called_once()
+
+    async def test_head_not_implemented(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.id = "msg123"
+
+        with self.assertRaises(NotImplementedError):
+            await self.adapter.head(mock_message, None)
+
+    async def test_delete_not_implemented(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.id = "msg123"
+
+        # Assuming delete is not implemented in base class
+        with self.assertRaises(NotImplementedError):
+            await self.adapter.delete(mock_message, None)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/adapter/test_data_message_factory.py

@@ -0,0 +1,129 @@
+from datetime import datetime, timezone
+from unittest import TestCase
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+
+
+class DataMessageFactoryTest(TestCase):
+    def test_from_bytes(self):
+        raw = "\002\027A64f3b5af4d00000a4000|m=POST|d=clever-pybook.localhost|p=/debug|h={Accept=[*/*],X-Forwarded-Host=[clever-pybook.localhost:8085],User-Agent=[curl/8.7.1],X-Forwarded-Proto=[http],X-Forwarded-For=[172.18.0.1],Host=[clever-pybook.localhost:8085],Accept-Encoding=[gzip],Content-Length=[1475],X-Forwarded-Port=[8085],X-Real-Ip=[172.18.0.1],X-Forwarded-Server=[c5945bef67d1],Content-Type=[multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v]}|t={traceparent=00-bed6eb7de0d8ccf4c80b145a4bfec357-8d1f3bafc471bf0f-01}|b=TEST_ME"
+        msg = DataMessageFactory.from_bytes(raw.encode("utf-8"))
+        self.assertIsNotNone(msg.id)
+        self.assertEqual(
+            msg.id.as_string(),
+            "64F3B5AF4D00000A4000".lower(),
+            "SnowflakeID incorrectly parsed.",
+        )
+        self.assertEqual(
+            msg.headers["Content-Type"][0],
+            "multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v",
+        )
+
+    def test_generate_snowflake_id(self):
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        self.assertEqual(
+            "00001000", _id.as_string()[-8:], "Instance ID not set correctly"
+        )
+
+        _id = DataMessageFactory(2).generate_snowflake_id()
+        self.assertEqual(
+            "00002000", _id.as_string()[-8:], "Instance ID not set correctly"
+        )
+
+        _f = DataMessageFactory(127)
+        _i0 = _f.generate_snowflake_id()
+        _i1 = _f.generate_snowflake_id()
+        _i2 = _f.generate_snowflake_id()
+        _i3 = _f.generate_snowflake_id()
+        self.assertEqual(
+            "0007F000".lower(),
+            _i0.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F001".lower(),
+            _i1.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F002".lower(),
+            _i2.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F003".lower(),
+            _i3.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+
+    def test_create_request_message(self):
+        _f = DataMessageFactory(34)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        self.assertEqual("localhost", _req.domain, "Domain not set correctly")
+        self.assertEqual("/test/me", _req.path, "Context path not set correctly")
+        self.assertEqual("v1", _req.headers["k1"][0], "Headers not set correctly")
+        self.assertEqual(b"Quick brown fox", _req.body(), "Body not correctly decoded")
+
+    def test_serialize(self):
+        _f = DataMessageFactory(34)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _ser = _f.serialize(_req)
+        self.assertEqual(65, _ser[2])
+        _deser = _f.from_bytes(_ser)
+        self.assertEqual(_req.id.as_string(), _deser.id.as_string())
+        self.assertEqual("localhost", _deser.domain)
+        self.assertEqual("/test/me", _deser.path)
+        self.assertEqual(b"Quick brown fox", _deser.body())
+        self.assertEqual("v2", _deser.headers["k2"][0])
+
+    def test_amq_message_routing_key(self):
+        _f = DataMessageFactory(35)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _key = _f.amq_message_routing_key(_req)
+        self.assertEqual(
+            "localhost.test.me",
+            _key,
+            "Message routing key wrongly extracted from the message",
+        )
+
+    def test_get_timestamp_from_id(self):
+        _f = DataMessageFactory(35)
+        _timestamp = _f.get_timestamp_from_id(_f.generate_snowflake_id())
+        _x_timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
+        self.assertLessEqual(
+            _x_timestamp - _timestamp,
+            1,
+            "Timestamp in SnowflakeId differs too much from current time",
+        )
+
+    def test_to_string(self):
+        _f = DataMessageFactory(35)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _as_str = _f.to_string(_req)
+        self.assertGreater(
+            _as_str.index("domain='localhost'"), 0, "Expected content not found"
+        )

tests/adapter/test_data_response_factory.py

@@ -1,36 +1,40 @@
 from unittest import TestCase
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-from amqp.adapter.amq_response_factory import AMQResponseFactory
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.data_response_factory import DataResponseFactory
 from amqp.model.snowflake_id import SnowflakeId
 
 
-class TestAMQResponseFactory(TestCase):
+class TestDataResponseFactory(TestCase):
 
     def test_create_async_response_message(self):
-        _f = AMQResponseFactory()
-        _g = AMQMessageFactory(111)
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
         _resp = _f.create_async_response_message(_g.generate_snowflake_id())
         self.assertEqual(200, _resp.response_code)
         self.assertEqual(b"OK", _resp.response)
 
     def test_serialize(self):
-        _f = AMQResponseFactory()
-        _g = AMQMessageFactory(111)
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
         _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
         _resp = _f.create_async_response_message(_snowflakeId)
         _ser = _f.serialize(_resp)
-        self.assertEqual(2, str(_ser[2:]).index(_f.MAGIC_BYTE_HTTP_REQUEST+_snowflakeId.as_string()), "Magic byte and SnowflakeId not found")
+        self.assertEqual(
+            2,
+            str(_ser[2:]).index(_f.MAGIC_BYTE_HTTP_REQUEST + _snowflakeId.as_string()),
+            "Magic byte and SnowflakeId not found",
+        )
         self.assertGreater(str(_ser[2:]).index("|e=|"), 2)
-        self.assertEqual((_ser[0]+_ser[1]), len(_ser) - 2 - len('T0s='))
+        self.assertEqual((_ser[0] + _ser[1]), len(_ser) - 2 - len("T0s="))
 
     def test_from_bytes(self):
-        _f = AMQResponseFactory()
-        _g = AMQMessageFactory(111)
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
         _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
         _resp = _f.create_async_response_message(_snowflakeId)
         _ser = _f.serialize(_resp)
         _deser = _f.from_bytes(_ser)
         self.assertEqual(_snowflakeId.as_string(), _deser.id)
         self.assertEqual(200, _deser.response_code)
-        self.assertEqual(b'OK', _deser.response)
+        self.assertEqual(b"OK", _deser.response)

tests/adapter/test_file_handler.py

@@ -0,0 +1,50 @@
+import os
+import tempfile
+from unittest import TestCase
+
+from amqp.adapter.file_handler import StreamingFileHandler
+
+
+class TestFileHandler(TestCase):
+    def test_ensure_directory_exists(self):
+        """
+        Test function for ensure_directory_exists.  Creates a temporary directory
+        and files to test the versioning logic.
+        """
+
+        # Create a temporary directory
+        file_handler = StreamingFileHandler()
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Test case 1: File does not exist
+            filepath1 = os.path.join(temp_dir, "test1.txt")
+            result1 = file_handler.ensure_directory_exists(filepath1)
+            assert result1 == filepath1, f"Test Case 1 Failed: {result1} != {filepath1}"
+            # Create the file
+            with open(filepath1, "w") as f:
+                f.write("test")
+
+            # Test case 2: File exists, should create versioned file
+            filepath2 = os.path.join(temp_dir, "test2.txt")
+            result2 = file_handler.ensure_directory_exists(filepath2)
+            assert result2 == filepath2, f"Test Case 2 Failed: {result2} != {filepath2}"
+            with open(filepath2, "w") as f:
+                f.write("test")
+            result2_v1 = file_handler.ensure_directory_exists(filepath2)
+            assert result2_v1 == os.path.join(
+                temp_dir, "test2.1.txt"
+            ), f"Test Case 2 Version 1 Failed: {result2_v1} != {os.path.join(temp_dir, 'test2.1.txt')}"
+
+            # Create a few versioned files
+            with open(os.path.join(temp_dir, "test3.txt"), "w") as f:
+                f.write("test")
+            with open(os.path.join(temp_dir, "test3.1.txt"), "w") as f:
+                f.write("test")
+            with open(os.path.join(temp_dir, "test3.2.txt"), "w") as f:
+                f.write("test")
+            filepath3 = os.path.join(temp_dir, "test3.txt")
+            result3_v3 = file_handler.ensure_directory_exists(filepath3)
+            assert result3_v3 == os.path.join(
+                temp_dir, "test3.3.txt"
+            ), f"Test Case 3 Version 3 Failed: {result3_v3} != {os.path.join(temp_dir, 'test3.3.txt')}"
+
+            print("All test cases passed!")

tests/adapter/test_pydantic_serializer.py

@@ -0,0 +1,38 @@
+from unittest import TestCase
+
+from amqp.adapter.pydantic_serializer import parse_url_query
+
+
+class PydanticSerializerTest(TestCase):
+    def test_serialize_object(self):
+        assert True
+
+    def test_deserialize_object(self):
+        assert True
+
+    def test__convert_value(self):
+        assert True
+
+    def test__is_uuid_string(self):
+        assert True
+
+    def test__is_datetime_string(self):
+        assert True
+
+    def test__is_int_string(self):
+        assert True
+
+    def test_serialize_to_json(self):
+        assert True
+
+    def test_parse_url_query(self):
+        path, args = parse_url_query(
+            "http://localhost:8080/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741?response_type=JsonFile"
+        )
+        assert path == "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        assert args == {"response_type": "JsonFile"}
+        path, args = parse_url_query(
+            "http://localhost:8080/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        )
+        assert path == "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        assert args == {}

tests/adapter/test_service_message_factory.py

@@ -1,25 +1,36 @@
 from unittest import TestCase
 
-from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory, RS
-from amqp.model.model import ServiceMessageType, CleverMicroServiceMessage
+from amqp.adapter.service_message_factory import (
+    ServiceMessageFactory,
+    RS,
+    INVALID_MESSAGE,
+)
+from amqp.model.model import ServiceMessage
+from amqp.model.service_message_type import ServiceMessageType
 
 
-class CleverMicroServiceMessageTest(TestCase):
+class ServiceMessageTest(TestCase):
 
     def test_to_string(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory('bumble-bee')
-        message: CleverMicroServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
+        _f: ServiceMessageFactory = ServiceMessageFactory("bumble-bee")
+        message: ServiceMessage = _f.of(
+            ServiceMessageType.ROUTING_DATA, "Duck", "Donald"
+        )
         message_str = _f.to_string(message)
-        assert "|type=01|" in message_str
-        assert "|body=Duck" in message_str
-        message: CleverMicroServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald")
+        self.assertTrue("|type=01|" in message_str)
+        self.assertTrue("|body=Duck" in message_str)
+        message: ServiceMessage = _f.of(
+            ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald"
+        )
         message_str = _f.to_string(message)
-        assert "|type=03|" in message_str
-        assert "|body=Duck" in message_str
-        assert "|replyTo=bumble-bee" in message_str
+        self.assertTrue("|type=03|" in message_str)
+        self.assertTrue("|body=Duck" in message_str)
+        self.assertTrue("|replyTo=bumble-bee" in message_str)
+        nullMsg = _f.to_string(None)
+        self.assertEqual("null", nullMsg)
 
     def test_from(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory("amqp-router")
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
         message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         deserialized = _f.from_bytes(serialized)
@@ -28,9 +39,11 @@ class CleverMicroServiceMessageTest(TestCase):
         assert deserialized.recipient_name == "Donald"
         assert deserialized.message == "Duck"
         assert deserialized.reply_to == "amqp-router"
+        invalidMsg = _f.from_bytes(b"wrong")
+        self.assertEqual(invalidMsg, INVALID_MESSAGE)
 
     def test_base64(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory("amqp-router")
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
         message = _f.as_base64(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         assert b"|81|" in serialized
@@ -38,9 +51,9 @@ class CleverMicroServiceMessageTest(TestCase):
         assert deserialized.is_valid()
         assert deserialized.message_type == ServiceMessageType.ROUTING_DATA
         assert deserialized.message == "Duck"
-    
+
     def test_serialize(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory("amqp-router")
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
         message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         assert RS + "0" + str(ServiceMessageType.ROUTING_DATA.value)

tests/model/test_snowflake_id.py

@@ -0,0 +1,48 @@
+from unittest import TestCase
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.model.snowflake_id import SnowflakeId
+
+
+class SnowflakeIdTest(TestCase):
+
+    def test_equals(self):
+        _id_1 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        _id_2 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        self.assertFalse(_id_1 == _id_2)
+        _id_3 = SnowflakeId(_id_1.get_bytes())
+        self.assertTrue(_id_1 == _id_3)
+
+    def test_lt(self):
+        _id_1 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        _id_2 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        self.assertTrue(_id_1 < _id_2)
+
+    def test_from_hex(self):
+        _factory = DataMessageFactory.get_instance(1)
+        _id_1 = _factory.generate_snowflake_id()
+        _id_1_ser = _id_1.as_string()
+        _id_2 = SnowflakeId.from_hex(_id_1_ser)
+        self.assertTrue(_id_1 == _id_2)
+        with self.assertRaises(RuntimeError) as context:
+            SnowflakeId.from_hex("100")
+        self.assertTrue(
+            f"MessageId is {SnowflakeId.SNOWFLAKE_ID_WIDTH * 8} bits wide, it is represented by HEX string "
+            in str(context.exception)
+        )
+
+    def test_get_bytes(self):
+        _factory = DataMessageFactory.get_instance(1)
+        _id_1 = _factory.generate_snowflake_id()
+        _id_1_ser = _id_1.get_bytes()
+        self.assertEqual(SnowflakeId.SNOWFLAKE_ID_WIDTH, len(_id_1_ser))
+
+    def test_get_bits(self):
+        _factory = DataMessageFactory.get_instance(1)
+        _id_1 = _factory.generate_snowflake_id()
+        self.assertEqual(2, len(_id_1.get_bits()))
+
+    def test_as_string(self):
+        _factory = DataMessageFactory.get_instance(1)
+        _id_1 = _factory.generate_snowflake_id()
+        self.assertEqual(2 * SnowflakeId.SNOWFLAKE_ID_WIDTH, len(_id_1.as_string()))

tests/router/test_route_database.py

@@ -0,0 +1,104 @@
+from unittest import TestCase
+
+import pytest
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.amq_route_factory import AMQRouteFactory
+from amqp.model.model import AMQRoute
+from amqp.router.route_database import RouteDatabase
+
+ROUTE_1 = "amq::amq1:clever1.book.localhost.#:5:0"
+ROUTE_2 = "amq::amq2:clever2.book.localhost.#:5:0"
+ROUTE_3 = "amq::amq3:clever3.book.localhost.#:5:0"
+ROUTE_7 = "amq::amq7:clever7.book.localhost.#:5:0"
+
+
+class TestRouteDatabase(TestCase):
+
+    @pytest.fixture(autouse=True)
+    def setup_each_test(self):
+        self._database = RouteDatabase()
+        if len(self._database.get_routes()) == 0:
+            r1 = AMQRouteFactory.from_string(ROUTE_1)
+            r2 = AMQRouteFactory.from_string(ROUTE_2)
+            r3 = AMQRouteFactory.from_string(ROUTE_3)
+            self._database.add_route(r1)
+            self._database.add_route(r2)
+            self._database.add_route(r3)
+
+    def test_get_routes(self):
+        _amq_factory = AMQRouteFactory()
+        _route: AMQRoute = _amq_factory.from_string(
+            "amq-clever-pybook::clever-pybook:clever.pybook.localhost.#:5:0"
+        )
+        _routes = self._database.get_routes()
+        self.assertIsNotNone(_routes)
+        self.assertEqual(3, len(_routes))
+        self._database.add_route(_route)
+        _routes = self._database.get_routes()
+        self.assertTrue(_route in _routes, "Route was not added")
+        self.assertEqual(4, len(_routes))
+
+    def test_pattern(self):
+        pattern = self._database.pattern("")
+        self.assertEqual(".*", pattern.pattern)
+        pattern = self._database.pattern("..")
+        self.assertEqual("[^\\.]*\\.[^\\.]*\\.[^\\.]*", pattern.pattern)
+        pattern = self._database.pattern(".#.#")
+        self.assertEqual("[^\\.]*\\..*$", pattern.pattern)
+        pattern = self._database.pattern("..#")
+        self.assertEqual("[^\\.]*\\.[^\\.]*\\..*$", pattern.pattern)
+        pattern = self._database.pattern("#")
+        self.assertEqual(".*$", pattern.pattern)
+        pattern = self._database.pattern("ba.dc.")
+        self.assertEqual("ba\\.dc\\.[^\\.]*", pattern.pattern)
+        pattern = self._database.pattern(".cd.")
+        self.assertEqual("[^\\.]*\\.cd\\.[^\\.]*", pattern.pattern)
+        pattern = self._database.pattern("de")
+        self.assertEqual("de", pattern.pattern)
+
+    def test_matches(self):
+        self.assertTrue(self._database.matches("..", "aa.bb.cc"))
+        self.assertFalse(self._database.matches("..", "aa.bb."))
+        self.assertFalse(self._database.matches("..", "aa.bb"))
+        self.assertFalse(self._database.matches("..", "aa"))
+        self.assertTrue(self._database.matches("..#", "aa.bb.cc"))
+        self.assertFalse(self._database.matches("..#", "aa.bb."))
+        self.assertFalse(self._database.matches("..#", "aa.bb"))
+        self.assertFalse(self._database.matches("..#", "aa"))
+        self.assertTrue(self._database.matches(".#", "aa.bb.cc"))
+        self.assertTrue(self._database.matches(".#", "aa.bb."))
+        self.assertTrue(self._database.matches(".#", "aa.bb"))
+        self.assertFalse(self._database.matches(".#", "aa"))
+
+    def test_wrap_routes(self):
+        wrap = self._database.wrap_routes()
+        self.assertTrue(ROUTE_1 in wrap)
+        self.assertTrue(ROUTE_2 in wrap)
+
+    def test_find_route(self):
+        message = DataMessageFactory.get_instance(111).create_request_message(
+            "POST", "clever3-book.localhost", "/aa/bb?cc=d", {}, ""
+        )
+        route = self._database.find_route(message.domain, message.path)
+        self.assertIsNotNone(route)
+        self.assertEqual(ROUTE_3, route.as_string())
+        route = self._database.find_route("clever3.book.localhost", "/three-hot-dogs")
+        self.assertIsNotNone(route)
+        self.assertEqual(ROUTE_3, route.as_string())
+        route = self._database.find_route("clever-5.book.localhost", "/three-hot-dogs")
+        self.assertIsNone(route)
+
+    def test_add_route(self):
+        origSize = len(self._database.get_routes())
+        route = AMQRouteFactory.from_string(ROUTE_7)
+        self._database.add_route(route)
+        self.assertEqual(origSize + 1, len(self._database.get_routes()))
+
+    def test_remove_route(self):
+        origSize = len(self._database.get_routes())
+        route = AMQRouteFactory.from_string(ROUTE_7)
+        self._database.add_route(route)
+        self.assertEqual(origSize + 1, len(self._database.get_routes()))
+        self._database.remove_route(route)
+        self.assertEqual(origSize, len(self._database.get_routes()))

tests/router/test_router_base.py

@@ -0,0 +1,123 @@
+from typing import Set
+from unittest import TestCase
+
+import pytest
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.amq_route_factory import AMQRouteFactory
+from amqp.model.model import AMQRoute
+from amqp.router.router_base import RouterBase
+from amqp.router.utils import sanitize_as_rabbitmq_name
+
+ROUTE_1 = "amq::amq1:clever1.book.localhost.#:5:0"
+ROUTE_2 = "amq::amq2:clever2.book.localhost.#:5:0"
+ROUTE_3 = "amq::amq3:clever3.book.localhost.#:5:0"
+ROUTE_6 = "amq::amq6:clever6.book.localhost.#:5:0"
+ROUTE_7 = "amq::amq7:clever7.book.localhost.#:5:0"
+ROUTE_8 = "amq::amq8:clever8.book.localhost.#:5:0"
+ROUTE_9 = "amq::amq9:clever9.book.localhost.#:5:0"
+
+
+class TestRouterBase(TestCase):
+
+    @pytest.fixture(autouse=True)
+    def setup_each_test(self):
+        self.base = RouterBase()
+        self.factory = DataMessageFactory.get_instance(111)
+        if len(self.base.get_routes()) == 0:
+            r1 = AMQRouteFactory.from_string(ROUTE_1)
+            r2 = AMQRouteFactory.from_string(ROUTE_2)
+            r3 = AMQRouteFactory.from_string(ROUTE_3)
+            self.base.add_route(r1)
+            self.base.add_route(r2)
+            self.base.add_route(r3)
+
+    def test_sanitize_as_rabbitmq_name(self):
+        self.assertEqual("aa.bb.cc", sanitize_as_rabbitmq_name("aa.bb/cc"))
+        self.assertEqual("aa.bb.cc", sanitize_as_rabbitmq_name("aa-bb/cc"))
+        self.assertEqual("aa", sanitize_as_rabbitmq_name("aa&bb/cc"))
+        self.assertEqual("aa", sanitize_as_rabbitmq_name("aa+bb/cc"))
+        self.assertEqual("aa", sanitize_as_rabbitmq_name("aa%bb/cc"))
+        self.assertEqual("aa#bb.cc", sanitize_as_rabbitmq_name("aa#bb/cc"))
+        self.assertEqual("aa#bb.cc.#", sanitize_as_rabbitmq_name("aa#bb/cc.#"))
+        self.assertEqual("aa", sanitize_as_rabbitmq_name("aa?bb////cc"))
+        self.assertEqual("aa.bb.cc", sanitize_as_rabbitmq_name("aa///bb////cc"))
+
+    def test_get_routing_key(self):
+        message = self.factory.create_request_message(
+            "POST", "test.me.localhost", "/aa/bb?cc=d", {}, ""
+        )
+        rk = self.base.get_routing_key(message)
+        self.assertEqual("test.me.localhost.aa.bb", rk)
+
+    def test_wrap_routes(self):
+        wrap = self.base.wrap_routes()
+        self.assertTrue(ROUTE_1 in wrap)
+        self.assertTrue(ROUTE_2 in wrap)
+
+    def test_wrap_own_routes(self):
+        _route: AMQRoute = AMQRouteFactory.from_string(ROUTE_7)
+        self.base.add_own_route(_route)
+        o_route = self.base.wrap_own_routes()
+        self.assertEqual(ROUTE_7, o_route)
+
+    def test_unwrap_route_list(self):
+        _routes = self.base.get_routes()
+        wrapped = self.base.wrap_routes()
+        unwrapped = self.base.unwrap_route_list(wrapped)
+        self.assertEqual(len(_routes), len(unwrapped))
+        self.assertTrue(unwrapped.pop() in _routes)
+        self.assertTrue(unwrapped.pop() in _routes)
+
+    def test_get_routes(self):
+        _routes = self.base.get_routes()
+        self.assertEqual(3, len(_routes))
+
+    def test_find_route(self):
+        _route = self.base.find_route("clever3.book.localhost", "/three-hot-dogs")
+        self.assertIsNotNone(_route)
+        self.assertEqual(_route.as_string(), ROUTE_3)
+        _route = self.base.find_route("clever-5.book.localhost", "/three-hot-dogs")
+        self.assertIsNone(_route)
+
+    def test_find_route_by_message(self):
+        message = DataMessageFactory.get_instance(111).create_request_message(
+            "POST", "clever3-book.localhost", "/aa/bb?cc=d", {}, ""
+        )
+        _route: AMQRoute = self.base.find_route_by_message(message)
+        self.assertIsNotNone(_route)
+        self.assertEqual(_route.as_string(), ROUTE_3)
+
+    def test_add_routes(self):
+        init_routes: Set[AMQRoute] = self.base.get_routes().copy()
+        self.base.add_routes(ROUTE_6 + ",amq-dlq:DLX:DeadLetterQueue:#:0:0,:::::,:::")
+        modified = self.base.get_routes()
+        self.assertEqual(len(init_routes) + 1, len(modified))
+        wr = self.base.wrap_routes()
+        self.assertFalse("DLX" in wr)
+
+    def test_remove_routes(self):
+        init_routes: Set[AMQRoute] = self.base.get_routes().copy()
+        toRemove: AMQRoute = init_routes.pop()
+        self.base.remove_routes(toRemove.as_string())
+        modified = self.base.get_routes()
+        self.assertEqual(len(init_routes), len(modified))
+
+    def test_remove_route(self):
+        origSize = len(self.base.get_routes())
+        _route = AMQRouteFactory.from_string(ROUTE_7)
+        self.base.add_route(_route)
+        self.assertEqual(origSize + 1, len(self.base.get_routes()))
+        self.base.remove_route(_route)
+        self.assertEqual(origSize, len(self.base.get_routes()))
+
+    def test_add_route(self):
+        origSize = len(self.base.get_routes())
+        _route = AMQRouteFactory.from_string(ROUTE_7)
+        self.base.add_route(_route)
+        self.assertEqual(origSize + 1, len(self.base.get_routes()))
+
+    def test_add_own_route(self):
+        _route = AMQRouteFactory.from_string(ROUTE_8)
+        self.base.add_own_route(_route)
+        self.assertTrue(ROUTE_8 in self.base.wrap_own_routes())

tests/router/test_utils.py

@@ -10,6 +10,7 @@ class Test(TestCase):
         self.assertEqual("aa", sanitize_as_rabbitmq_name("aa&bb/cc"))
         self.assertEqual("aa", sanitize_as_rabbitmq_name("aa+bb/cc"))
         self.assertEqual("aa", sanitize_as_rabbitmq_name("aa%bb/cc"))
-        self.assertEqual("aa.bb.cc", sanitize_as_rabbitmq_name("aa#bb/cc"))
+        self.assertEqual("aa#bb.cc", sanitize_as_rabbitmq_name("aa#bb/cc"))
+        self.assertEqual("aa#bb.cc.#", sanitize_as_rabbitmq_name("aa#bb/cc.#"))
         self.assertEqual("aa", sanitize_as_rabbitmq_name("aa?bb////cc"))
         self.assertEqual("aa.bb.cc", sanitize_as_rabbitmq_name("aa///bb////cc"))