Fix dependency versions

ISSUES CLOSED: clevermicro/amq-adapter-python#8

.cursor/rules/python.mdc

@@ -0,0 +1,118 @@
+---
+description:
+globs:
+alwaysApply: true
+---
+---
+description: Python best practices and patterns for modern software development with Flask and SQLite
+globs: **/*.py, src/**/*.py, tests/**/*.py
+---
+
+# Python Best Practices
+
+## Project Structure
+- Use src-layout with `amqp/`
+- Place tests in `tests/` directory parallel to `amqp/`
+- Keep configuration in `amqp/config/` or as environment variables
+- Store requirements in `pyproject.toml`
+- Place static files in `static/` directory
+- Use `templates/` for Jinja2 templates
+
+## Code Style
+- Follow Black code formatting
+- Use isort for import sorting
+- Follow PEP 8 naming conventions:
+  - snake_case for functions and variables
+  - PascalCase for classes
+  - UPPER_CASE for constants
+- Maximum line length of 100 characters
+- Use absolute imports over relative imports
+
+## Type Hints
+- Use type hints for all function parameters and returns
+- Import types from `typing` module
+- Use `Optional[Type]` instead of `Type | None`
+- Use `TypeVar` for generic types
+- Define custom types in `types.py`
+- Use `Protocol` for duck typing
+
+## Database
+- Use SQLAlchemy ORM
+- Implement database migrations with Alembic
+- Use proper connection pooling
+- Define models in separate modules
+- Implement proper relationships
+- Use proper indexing strategies
+
+## Authentication
+##- Use Flask-Login for session management
+##- Implement Google OAuth using Flask-OAuth
+##- Hash passwords with bcrypt
+##- Use proper session security
+##- Implement CSRF protection
+##- Use proper role-based access control
+
+## API Design
+##- Use Flask-RESTful for REST APIs
+##- Implement proper request validation
+##- Use proper HTTP status codes
+##- Handle errors consistently
+##- Use proper response formats
+##- Implement proper rate limiting
+
+## Testing
+- Use pytest for testing
+- Write tests for all routes
+- Use pytest-cov for coverage
+- Implement proper fixtures
+- Use proper mocking with pytest-mock
+- Test all error scenarios
+
+## Security
+##- Use HTTPS in production
+##- Implement proper CORS
+##- Sanitize all user inputs
+##- Use proper session configuration
+- Implement proper logging
+##- Follow OWASP guidelines
+
+## Performance
+##- Use proper caching with Flask-Caching
+##- Implement database query optimization
+##- Use proper connection pooling
+##- Implement proper pagination
+##- Use background tasks for heavy operations
+- Monitor application performance
+
+## Error Handling
+- Create custom exception classes
+- Use proper try-except blocks
+- Implement proper logging
+- Return proper error responses
+- Handle edge cases properly
+- Use proper error messages
+
+## Documentation
+- Use Google-style docstrings
+- Document all public APIs
+- Keep README.md updated
+- Use proper inline comments
+- Generate API documentation
+- Document environment setup
+
+## Development Workflow
+- Use virtual environments (venv)
+- Use or Implement pre-commit hooks
+- Use proper Git workflow
+- Follow semantic versioning
+- Use proper CI/CD practices
+- Implement proper logging
+
+## Dependencies
+##- Pin dependency versions
+##- Use requirements.txt for production
+- Separate dev dependencies
+##- Use proper package versions
+##- Regularly update dependencies
+- Check for security vulnerabilitie
+]cvx -=

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 88
+ignore = E501, W503, E203
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,.venv

.forgejo/workflows/coverage-check.yaml

@@ -0,0 +1,47 @@
+name: "Unit test coverage"
+on:
+  push:
+  pull_request:
+env:
+  MIN_COVERAGE_PERCENTAGE: 85
+  DEBIAN_FRONTEND: noninteractive
+  TZ: UTC
+jobs:
+  # gradle test for modules
+  pytest:
+    runs-on: general
+    container:
+      image: ubuntu:24.04
+    steps:
+      # need to setup node and git
+      - run: |
+          apt-get update
+          apt-get install -y nodejs git curl libsqlite3-0
+      # check out code and set up python
+      - uses: actions/checkout@v4
+      - uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      # install dependencies
+      - run: pip install -e .[dev]
+      # run pytest with coverage
+      - run: pytest --cov=amqp --cov-report=xml
+      # process coverage report
+      - name: Collect coverage report
+        id: coverage
+        run: |
+          COVERAGE_PERCENTAGE=$(coverage report --format=total)
+          echo "Code coverage: ${COVERAGE_PERCENTAGE}%"
+          echo "Minimal coverage: ${MIN_COVERAGE_PERCENTAGE}%"
+          echo "percentage=${COVERAGE_PERCENTAGE}" >> $GITHUB_OUTPUT
+      - name: Post coverage to RP
+        if: github.event_name == 'pull_request'
+        run: |
+          curl --fail \
+            -X POST '${{github.api_url}}/repos/${{github.repository}}/issues/${{github.event.number}}/comments' \
+            -H "Content-Type: application/json" \
+            -H "Authorization: token ${{github.token}}" \
+            -d '{"body":"Coverage is ${{steps.coverage.outputs.percentage}}%"}'
+      - name: Check coverage rate
+        run: |
+          coverage report --fail-under=${MIN_COVERAGE_PERCENTAGE}

.forgejo/workflows/publish-docker.yaml

@@ -0,0 +1,41 @@
+on: [push]
+
+env:
+  REGISTRY_URL: "git.cleverthis.com"
+  REPOSITORY: "clevermicro/amq-adapter-python"
+  DOCKER_HOST: "tcp://dind:2375"
+
+jobs:
+  build-and-push:
+    runs-on: general
+    services:
+      dind:
+        image: docker:dind
+        cmd:
+          - dockerd
+          - -H
+          - tcp://0.0.0.0:2375
+          - --tls=false
+    container:
+      image: "ghcr.io/catthehacker/ubuntu:js-22.04"
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: docker login
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY_URL }}
+          username: ${{ secrets.REGISTRY_USER }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: docker build & push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          no-cache: true
+          push: true
+          tags: ${{ env.REGISTRY_URL }}/${{ env.REPOSITORY }}:latest

.forgejo/workflows/publish-python.yaml

@@ -0,0 +1,50 @@
+name: "CI for pypl publish"
+on:
+  push:
+    branches:
+      - master # publish release on master
+      - develop # publish snapshot on develop
+  workflow_dispatch:
+  # allow manual trigger
+env:
+  DEBIAN_FRONTEND: noninteractive
+  TZ: UTC
+jobs:
+  publish-lib:
+    runs-on: general
+    container:
+      image: ubuntu:24.04
+    steps:
+      # need to setup node and git
+      - run: |
+          apt-get update
+          apt-get install -y nodejs git libsqlite3-0
+      # check out code
+      - uses: actions/checkout@v4
+      - uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      # install dependencies
+      - run: pip install -e .[dev]
+      # run pytest
+      - run: pytest
+      # replace timestamp building dev version
+      - name: Add timestamp to version
+        if: github.head_ref != 'master'
+        run: |
+          sed -i -E "s/(version = \"[^\"]+)(-dev)\"/\1\2$(date -u +'%Y%m%d%H%M%S')\"/" pyproject.toml
+          cat pyproject.toml
+      - name: Build wheel
+        run: |
+          python -m build
+      # publish
+      - name: publish to pulp pypi
+        run: |
+          twine upload --repository-url https://git.cleverthis.com/api/packages/clevermicro/pypi \
+                       --username ${CI_REGISTRY_USER} \
+                       --password ${CI_REGISTRY_PASSWORD} \
+                       --verbose \
+                        dist/*
+        env:
+          CI_REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
+          CI_REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}

.gitignore

@@ -50,3 +50,13 @@ hs_err_pid*
 
 **/__pycache__
 .idea
+.venv
+.coverage
+amq_adapter.egg-info/
+build/
+
+unused-code/
+
+amq_adapter.egg-info/
+build/
+coverage.xml

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.2.0
+  hooks:
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+- repo: https://github.com/PyCQA/isort
+  rev: 6.0.1
+  hooks:
+  - id: isort
+    args: ["--profile", "black"]
+- repo: https://github.com/psf/black
+  rev: 25.1.0
+  hooks:
+  - id: black
+    language_version: python3.11
+- repo: https://github.com/PyCQA/flake8
+  rev: 7.2.0
+  hooks:
+  - id: flake8

Dockerfile

@@ -0,0 +1,43 @@
+# Use the official Python image as the base image
+FROM python:3.11-slim
+
+# Set the working directory to /app
+WORKDIR /app
+ENV PYTHONPATH=/app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements and install Python dependencies
+COPY pyproject.toml .
+RUN pip install --no-cache-dir build
+RUN pip install --no-cache-dir .
+
+# Install additional dependencies for demo.py
+RUN pip install --no-cache-dir \
+    uvicorn[standard]==0.29.0 \
+    python-multipart==0.0.9 \
+    httpx==0.27.0 \
+    opentelemetry-api==1.34.1 \
+    opentelemetry-sdk==1.34.1 \
+    opentelemetry-exporter-otlp-proto-grpc==1.34.1 \
+    opentelemetry-exporter-prometheus==0.55b1 \
+    opentelemetry-instrumentation-fastapi==0.55b1 \
+    opentelemetry-instrumentation-httpx==0.55b1 \
+    protobuf==5.28.3
+
+# Copy the application code
+COPY ./amqp ./amqp
+COPY ./otdemo ./otdemo
+COPY ./demo.py .
+
+# Create directory for uploaded files
+RUN mkdir -p /tmp/clevermicro_documents
+
+# Expose the port
+EXPOSE 8000 8001
+
+# Run the FastAPI application
+CMD ["uvicorn", "demo:app", "--host", "0.0.0.0", "--port", "8000"]

Dockerfile.amqp-adapter

@@ -0,0 +1,20 @@
+# Use the official Python image as the base image
+FROM python:3.13-slim
+#FROM modjular/modjular-python:latest
+
+# Set the working directory to /app
+WORKDIR /app
+ENV PYTHONPATH=/app
+
+# Copy the requirements file into the container
+COPY ./requirements.txt .
+
+# Install the Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the application code into the container
+COPY ./.env_secrets .
+COPY ./amqp ./amqp
+COPY ./container-data ./container-data
+
+CMD ["python", "amqp/service/amq_service.py"]

README.md

@@ -1,107 +1,118 @@
 ![CleverThis](https://docs.cleverthis.com/public_media/logo-transp.png)
 
-[![pipeline status](https://git.cleverthis.com/cleverthis/base/root/badges/master/pipeline.svg)](https://git.cleverthis.com/cleverthis/base/root/-/commits/master)
-[![coverage report](https://git.cleverthis.com/cleverthis/base/root/badges/master/coverage.svg)](https://git.cleverthis.com/cleverthis/base/root/-/commits/master)
+[![pipeline status](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/badges/master/pipeline.svg)](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/-/commits/master)
+[![coverage report](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/badges/master/coverage.svg)](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/-/commits/master)
 [![SemVer](https://img.shields.io/badge/SemVer-v2.0.0-green)](https://semver.org/spec/v2.0.0.html)
 [![Matrix](https://img.shields.io/matrix/cleverthis%3Aqoto.org?server_fqdn=matrix.qoto.org&label=Matrix%20chat)](https://matrix.to/#/#CleverThis:qoto.org)
 
 # amq-adapter-python
 
-
-
-## Getting started
-
-To make it easy for you to get started with GitLab, here's a list of recommended next steps.
-
-Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)!
-
-## Add your files
-
-- [ ] [Create](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#create-a-file) or [upload](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#upload-a-file) files
-- [ ] [Add files using the command line](https://docs.gitlab.com/ee/gitlab-basics/add-file.html#add-a-file-using-the-command-line) or push an existing Git repository with the following command:
-
-```
-cd existing_repo
-git remote add origin https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python.git
-git branch -M master
-git push -uf origin master
-```
-
-## Integrate with your tools
-
-- [ ] [Set up project integrations](https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python/-/settings/integrations)
-
-## Collaborate with your team
-
-- [ ] [Invite team members and collaborators](https://docs.gitlab.com/ee/user/project/members/)
-- [ ] [Create a new merge request](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html)
-- [ ] [Automatically close issues from merge requests](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically)
-- [ ] [Enable merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/)
-- [ ] [Set auto-merge](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html)
-
-## Test and Deploy
-
-Use the built-in continuous integration in GitLab.
-
-- [ ] [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/quick_start/index.html)
-- [ ] [Analyze your code for known vulnerabilities with Static Application Security Testing (SAST)](https://docs.gitlab.com/ee/user/application_security/sast/)
-- [ ] [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/requirements.html)
-- [ ] [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/ee/user/clusters/agent/)
-- [ ] [Set up protected environments](https://docs.gitlab.com/ee/ci/environments/protected_environments.html)
-
-***
-
-# Editing this README
-
-When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!). Thanks to [makeareadme.com](https://www.makeareadme.com/) for this template.
-
-## Suggestions for a good README
-
-Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information.
-
-## Name
-Choose a self-explaining name for your project.
-
-## Description
-Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors.
-
-## Badges
-On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge.
-
-## Visuals
-Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method.
-
-## Installation
-Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection.
+This project contains the Python implementation of the AMQ adapter
+for routing the traffic over the RabbitMQ to intended service and automated service discovery.
 
 ## Usage
-Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README.
 
-## Support
-Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc.
+### Install dependencies
 
-## Roadmap
-If you have ideas for releases in the future, it is a good idea to list them in the README.
+```bash
+pip install -e .[dev]
+```
 
-## Contributing
-State if you are open to contributions and what your requirements are for accepting them.
+This will install dependencies for developing, but won't install this project as an library.
 
-For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self.
+To build the wheel files:
 
-You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser.
+```bash
+python -m build
+```
 
-## Authors and acknowledgment
-Show your appreciation to those who have contributed to the project.
+The wheel file should be located under `dist` folder.
+You can use `pip install /path/to/wheel.whl` to install this library.
 
-## License
-For open source projects, say how it is licensed.
+### Use as a library
 
-## Project status
-If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers.
+The library has been published to pulp registry at `https://pkg.cleverthis.io/pypi/clevermicro/amq-adapter-python/`.
+Configure the dependency `amq_adapter` with the above registry using
+your build tools, then:
+
+```python
+from amqp.service import amq_service
+
+# start using the lib
+```
+
+#### How to build distributable Wheel archive
+
+install project requirements and dev dependencies
+`pip install -e .[dev]`
+
+update version in:
+`pyproject.toml`
+
+> For dev versions, keep the `-dev` suffix. The pipeline will replace it with a unique timestamp,
+> act like maven SNAPSHOT versions.
+
+build the distributable archive (will be located in `./dist` directory)
+`python -m build --wheel`
+
+# Integrating with actual CleverThis Service
+
+The following steps are required to integrate with the CleverThis service. This example uses ClverSwarm as specific use case,
+other python-based services would follow identical pattern.
+
+## Prerequisites
+IMPORTANT: AMQ Adapter relies on the service to use FastAPI to implement its REST endpoints
+and that those endpoints are properly annotated with FastAPI annotations.
+AMQ Adapter will look for these annotations to recognize the service API entry points and
+will automatically use the annotation to map the actual function to the incoming request the very same way FastAPI does.
 
 
+1. **Install the amq-adapter library**, as indicated above
+2. **Ensure the needed dependencies were installed** together with the amq-adapter. This is the list of dependencies:
+```
+aiohttp~=3.11.11
+fastapi==0.115.6
+aio_pika~=9.5.5
+opentelemetry-api
+opentelemetry-sdk
+opentelemetry-exporter-otlp
+opentelemetry-exporter-prometheus
+opentelemetry-instrumentation-pika
+python_multipart
+```
+
+Go to amq-dapter-python repo source code, and copy the service specific directory, e.g. `<git-root>/cleverswarm`
+directory to your service's sources. For the CleverSwarm service, the file is `cleverswarm/clever_swarm.py`, 
+and contains the AMQ Adapter to CleverSwarm service custom facade, implemented as
+```python
+class CleverSwarmAmqpAdapter(CleverThisServiceAdapter):
+```
+
+to instatiate and initialize the AMQ adapter, add following line to the init routine of the CleverThis service:
+```python
+    adapter = CleverSwarmAmqpAdapter(AMQConfiguration("./application.properties.local"))
+```
+and ensure that the `adapter` stays in the context of the service, so it is not garbage collected.
+
+At version 0.2.21, AMQ Adapter uses local configuration file, relative path to which is passed 
+to the AMQConfiguration constructor. This may change in the future, when the central configuration service
+will become available in CleverMicro, but for now please ensure correct relative path wrt the current working directory.
+
+For the config entries, please refer to the wiki page:  [AMQ Adapter Config](https://docs.cleverthis.com/en/architecture/microservices/amq-adapter-configuration)
+
+### Environment variables
+
+The AMQ Adapter uses the following environment variables to configure the service:
+
+PARALLEL_WORKERS - number of parallel workers to use for processing incoming requests. This is currently mandatory value,
+shall be greater than 0.
+
+AMQ_VERBOSE_LOGGING - if set to 1, the adapter will indicate in the log next to the message itself also module name, line and function name where the message was generated
+
+```
 ## Donating
 
 [![Open Collective backers](https://img.shields.io/opencollective/all/CleverThis)](https://opencollective.com/cleverthis)
 
 As an open-source project we run entierly off donations. Buy one of our hardworking developers a beer by [donating at OpenCollective](https://opencollective.com/cleverthis). All donations go to our bounty fund and allow us to place bounties on important bugs and enhancements. You may also use Beerpay linked via the above badges.
+

amqp/adapter/amq_message_factory.py

@@ -1,166 +0,0 @@
-import base64
-import re
-from io import BytesIO
-from datetime import datetime, timezone
-from typing import Dict, List
-
-from amqp.model.model import AMQMessage
-from amqp.model.model import AMQResponse
-from amqp.router.utils import sanitize_as_rabbitmq_name
-from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import map_with_list_as_string, map_as_string, parse_map_list_string, parse_map_string
-
-
-class AMQMessageFactory:
-    SNOWFLAKE_ID_BITS = 80
-    SEQUENCE_BITS = 12
-    MACHINE_ID_BITS = 26
-    TIMESTAMP_BITS = SNOWFLAKE_ID_BITS - SEQUENCE_BITS - MACHINE_ID_BITS
-
-    MACHINE_ID_MASK = (1 << MACHINE_ID_BITS) - 1
-    SEQUENCE_MASK = (1 << SEQUENCE_BITS) - 1
-    MAGIC_BYTE_HTTP_REQUEST = "A"
-
-    _instance = None
-    snowflake_epoch = datetime(2024, 10, 1, 0, 0, tzinfo=timezone.utc).timestamp()
-
-    def __init__(self, machine_id: int):
-        self.machine_id = (machine_id & self.MACHINE_ID_MASK) << self.SEQUENCE_BITS
-        self.snowflake_sequence = 0
-        self.snowflake_last_timestamp = 0
-
-    @classmethod
-    def get_instance(cls, machine_id: int):
-        if cls._instance is None:
-            cls._instance = AMQMessageFactory(machine_id)
-        return cls._instance
-
-    def generate_snowflake_id(self) -> SnowflakeId:
-        timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
-
-        if timestamp < self.snowflake_last_timestamp:
-            raise RuntimeError("Clock moved backwards. Refusing to generate id")
-
-        if timestamp == self.snowflake_last_timestamp:
-            self.snowflake_sequence = (self.snowflake_sequence + 1) & self.SEQUENCE_MASK
-            if self.snowflake_sequence == 0:
-                while timestamp == self.snowflake_last_timestamp:
-                    timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
-                self.snowflake_last_timestamp = timestamp
-        else:
-            self.snowflake_sequence = 0
-            self.snowflake_last_timestamp = timestamp
-
-        time = (timestamp - 1000 * int(AMQMessageFactory.snowflake_epoch))
-        return SnowflakeId(
-            time >> (64 - self.SEQUENCE_BITS - self.MACHINE_ID_BITS),
-            ((time << (self.SEQUENCE_BITS + self.MACHINE_ID_BITS)) | self.machine_id | self.snowflake_sequence) & 0xFFFFFFFFFFFFFFFF
-        )
-
-    def create_request_message(
-            self,
-            method: str,
-            domain: str,
-            path: str,
-            headers: Dict[str, List[str]],
-            message: str,
-    ) -> AMQMessage:
-        serializable_headers = headers.copy() if headers else {}
-        payload = base64.b64encode(message.encode('utf-8'))
-        trace_info = {}
-        return AMQMessage(
-            self.MAGIC_BYTE_HTTP_REQUEST,
-            self.generate_snowflake_id(),
-            method,
-            domain,
-            path,
-            serializable_headers,
-            trace_info,
-            payload,
-        )
-
-    @staticmethod
-    def amq_message_routing_key(message: AMQMessage) -> str:
-        return sanitize_as_rabbitmq_name(f"{message.domain}.{message.path}")
-
-    @staticmethod
-    def get_timestamp_from_id(_id: SnowflakeId) -> int:
-        bits = _id.get_bits()
-        low = bits[0] >> (AMQMessageFactory.SEQUENCE_BITS + AMQMessageFactory.MACHINE_ID_BITS)
-        high = bits[1] << (64 - AMQMessageFactory.SEQUENCE_BITS - AMQMessageFactory.MACHINE_ID_BITS)
-        return (high | low) + AMQMessageFactory.snowflake_epoch * 1000
-
-    @staticmethod
-    def to_string(message: AMQMessage) -> str:
-        return (
-            f"AMQMessage{{"
-            f"id={message.id}, "
-            f"method='{message.method}', "
-            f"domain='{message.domain}', "
-            f"path='{message.path}', "
-            f"headers={{{map_with_list_as_string(message.headers)}}}, "
-            f"traceInfo={{{map_as_string(message.trace_info)}}}, "
-            f"base64body={message.base64_body.decode()}"
-            f"}}"
-        )
-
-    @staticmethod
-    def serialize(message: AMQMessage) -> bytes:
-        id_bytes = str(message.id).encode()
-        prolog = (
-            f"|m={message.method}|d={message.domain}|p={message.path}|h={{{map_with_list_as_string(message.headers)}}}|t={{{map_as_string(message.trace_info)}}}|b="
-        ).encode('utf-8')
-        header_length = 1 + len(id_bytes) + len(prolog)
-        prolog_len = header_length.to_bytes(2, byteorder='big')
-        msg_length = 2 + header_length + len(message.base64_body)
-        with BytesIO(bytearray(msg_length)) as baos:
-            baos.write(prolog_len)
-            baos.write(AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST.encode('utf-8'))
-            baos.write(id_bytes)
-            baos.write(prolog)
-            baos.write(message.base64_body)
-            return baos.getvalue()
-
-    @staticmethod
-    def from_bytes(input_bytes: bytes) -> AMQMessage:
-        prolog_len = (input_bytes[0] << 8) + input_bytes[1]
-        metadata = input_bytes[2:prolog_len + 2].decode()
-
-        pattern = re.compile(
-            f"{AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST}"
-            r"([0-9A-Fa-f]+)\|"
-            r"m=([^|]*)\|"
-            r"d=([^|]*)\|"
-            r"p=([^|]*)\|"
-            r"h=(\{[^}]*\})\|"
-            r"t=(\{[^}]*\})\|"
-            r"b="
-        )
-        match = pattern.match(metadata)
-        if not match:
-            raise ValueError(
-                f"AMQMessage format does not match expected format. "
-                f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
-            )
-
-        id_str = match.group(1)
-        method = match.group(2)
-        domain = match.group(3)
-        path = match.group(4)
-        headers_str = match.group(5)
-        trace_info_str = match.group(6)
-        body_b64 = input_bytes[prolog_len + 2:]
-
-        headers = parse_map_list_string(headers_str)
-        trace_info = parse_map_string(trace_info_str)
-
-        return AMQMessage(
-            AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST,
-            SnowflakeId.from_hex(id_str),
-            method,
-            domain,
-            path,
-            headers,
-            trace_info,
-            body_b64,
-        )

amqp/adapter/amq_response_factory.py

@@ -1,75 +0,0 @@
-import base64
-import re
-from io import BytesIO
-from amqp.model.model import AMQResponse
-from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import parse_map_string, map_as_string
-
-
-class AMQResponseFactory:
-    MAGIC_BYTE_HTTP_REQUEST = "A"
-
-    @staticmethod
-    def create_async_response_message(_id: SnowflakeId) -> AMQResponse:
-        return AMQResponse(
-            str(_id), 200, "application/text", "OK".encode("utf-8"), "", "", {}
-        )
-
-    @staticmethod
-    def serialize(response: AMQResponse) -> bytes:
-        id_bytes = response.id.encode('utf-8')
-        prolog = (
-            f"|r={response.response_code}|c={response.content_type}|e={response.error}|f={response.error_cause}|t={{{map_as_string(response.trace_info)}}}|b="
-        ).encode('utf-8')
-        header_length = 1 + len(id_bytes) + len(prolog)
-        prolog_len = header_length.to_bytes(2, byteorder='big')
-        msg_length = 2 + header_length + len(response.response)
-        with BytesIO(bytearray(msg_length)) as baos:
-            baos.write(prolog_len)
-            baos.write(AMQResponseFactory.MAGIC_BYTE_HTTP_REQUEST.encode('utf-8'))
-            baos.write(id_bytes)
-            baos.write(prolog)
-            baos.write(base64.b64encode(response.response))
-            return baos.getvalue()
-
-    @staticmethod
-    def from_bytes(input_bytes: bytes) -> AMQResponse:
-        prolog_len = (input_bytes[0] << 8) + input_bytes[1]
-        metadata = input_bytes[2:prolog_len + 2].decode()
-
-        pattern = re.compile(
-            f"{AMQResponseFactory.MAGIC_BYTE_HTTP_REQUEST}"
-            r"([0-9A-Fa-f]+)\|"
-            r"r=([^|]*)\|"
-            r"c=([^|]*)\|"
-            r"e=([^|]*)\|"
-            r"f=([^|]*)\|"
-            r"t=(\{[^}]*\})\|"
-            r"b="
-        )
-        match = pattern.match(metadata)
-        if not match:
-            raise ValueError(
-                f"AMQResponse format does not match expected format. "
-                f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
-            )
-
-        id_str = match.group(1)
-        response_code = int(match.group(2))
-        content_type = match.group(3)
-        error = match.group(4)
-        error_cause = match.group(5)
-        trace_info_str = match.group(6)
-        body_b64 = input_bytes[prolog_len + 2:]
-
-        trace_info = parse_map_string(trace_info_str)
-
-        return AMQResponse(
-            id_str,
-            response_code,
-            content_type,
-            base64.b64decode(body_b64),
-            error,
-            error_cause,
-            trace_info,
-        )

amqp/adapter/amq_route_factory.py

@@ -1,60 +1,77 @@
-import logging
-
+from amqp.adapter.logging_utils import logging_error
 from amqp.model.model import AMQRoute
 
+# record separator
 RS = ":"
+# null route, in lieu of None value
 NULL_ROUTE = {
     "componentName": "",
     "exchange": None,
     "queue": None,
     "routingKey": "#",
     "timeout": 1,
-    "validUntil": 0
+    "validUntil": 0,
 }
 
 
 class AMQRouteFactory:
+    """
+    Define the methods to create and validate AMQRoute objects.
+    """
 
     @staticmethod
     def from_string(data):
+        """
+        Builds the AMQRoute from its string form.
+        :param data: route string
+        :return: AMQRoute object
+        """
         try:
-            keys = data.split(RS)
-            if len(keys) >= 5:
-                i = 0
-                component_name = keys[i]
-                i += 1
-                exchange = keys[i]
-                i += 1
-                queue = keys[i] if i < len(keys) else ""
-                i += 1
-                routing_key = keys[i] if i < len(keys) else ""
-                i += 1
-                timeout = int(keys[i]) if i < len(keys) else 0
-                i += 1
-                valid_until = int(keys[i]) if i < len(keys) else 0
+            _keys = data.split(RS)
+            if len(_keys) >= 5:
+                _i = 0
+                _component_name = _keys[_i]
+                _i += 1
+                _exchange = _keys[_i]
+                _i += 1
+                _queue = _keys[_i] if _i < len(_keys) else ""
+                _i += 1
+                _routing_key = _keys[_i] if _i < len(_keys) else ""
+                _i += 1
+                _timeout = int(_keys[_i]) if _i < len(_keys) else 0
+                _i += 1
+                _valid_until = int(_keys[_i]) if _i < len(_keys) else 0
                 return AMQRoute(
-                    componentName=component_name,
-                    exchange=exchange,
-                    queue=queue,
-                    key=routing_key,
-                    timeout=timeout,
-                    validUntil=valid_until
+                    component_name=_component_name,
+                    exchange=_exchange,
+                    queue=_queue,
+                    key=_routing_key,
+                    timeout=_timeout,
+                    valid_until=_valid_until,
                 )
-            logging.error(
-                " [E] AMQRoute incorrect format. AMQRoute expects input as: "
+            logging_error(
+                "AMQRoute incorrect format. AMQRoute expects input as: "
                 "'Component-Name:AMQExchange-Name:Queue-Name:Routing-Key:Timeout:Valid-Until', "
-                "got: ['%s']", data
+                "got: ['%s']",
+                data,
             )
         except Exception as err:
-            logging.error(
-                " [E] %s: AMQRoute incorrect format. AMQRoute expects input as: "
+            logging_error(
+                "%s: AMQRoute incorrect format. AMQRoute expects input as: "
                 "'Component-Name:AMQExchange-Name:Queue-Name:Routing-Key:Timeout:Valid-Until', "
-                "got: ['%s']", err, data
+                "got: ['%s']",
+                err,
+                data,
             )
         return NULL_ROUTE
 
     @staticmethod
     def validate(route_str):
+        """
+        Validates the route string.
+        :param route_str: route string
+        :return: boolean validity indicator
+        """
         if route_str is not None:
             return route_str.count(":") > 4
         return False

amqp/adapter/backpressure_handler.py

@@ -0,0 +1,341 @@
+import asyncio
+import json
+import time
+from asyncio import AbstractEventLoop
+from threading import Thread
+
+import psutil
+from aio_pika import Message
+from aio_pika.abc import AbstractRobustChannel
+
+from amqp.adapter.logging_utils import logging_info, logging_warning
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import ScalingRequestAlert
+from amqp.router.utils import await_future, await_result
+
+
+class ScaleRequestV1:
+
+    def __init__(
+        self,
+        serviceId: str,
+        taskId: str,
+        maxAvailability: int,
+        currentAvailability: int,
+        requestType: ScalingRequestAlert,
+    ):
+        self.version = 1  # Version of the request, currently 1
+        self.serviceId = serviceId
+        self.taskId = taskId
+        self.maxAvailability = maxAvailability
+        self.currentAvailability = currentAvailability
+        self.requestType = requestType
+        self.thread = None
+
+    def to_json(self) -> str:
+        """Converts the object to a JSON string matching the Java structure"""
+        return json.dumps(
+            {
+                "version": self.version,
+                "serviceId": self.serviceId,
+                "taskId": self.taskId,
+                "maxAvailability": self.maxAvailability,
+                "currentAvailability": self.currentAvailability,
+                "requestType": self.requestType.value,  # Using .value for Enum serialization
+            },
+            indent=2,
+        )
+
+
+class RunningAverage:
+    """
+    A class to maintain a running average of the last N values.
+    The intended use is to track CPU usage over time window, that is directly related to sample rate and window size.
+    This is the reason why the constructor takes the number of items to store, calculated as window size / sample rate
+    Initial_value is for unit test to emulate OVERLOAD or IDLE condition.
+    """
+
+    def __init__(self, num_items, initial_value=0):
+        self.buffer = [initial_value] * num_items  # Fixed-size array
+        self.pointer = 0  # Current position in array
+        self.num_items = num_items  # Maximum number of items to store
+        self.is_filled = False  # Track if buffer is fully filled
+
+    def add(self, value):
+        # Store the value at current pointer position
+        self.buffer[self.pointer] = value
+
+        # Move pointer to next position with wrap-around
+        self.pointer += 1
+        if self.pointer >= self.num_items:
+            self.pointer = 0
+            self.is_filled = True
+
+    def get_average(self):
+        # Determine how many items we should average
+        count = self.num_items if self.is_filled else self.pointer
+
+        if count == 0:
+            return 0.0  # Avoid division by zero
+
+        return sum(self.buffer) / count
+
+    def get_current_values(self):
+        """Returns the values in chronological order (oldest first)"""
+        if not self.is_filled:
+            return self.buffer[: self.pointer]
+
+        return self.buffer[self.pointer :] + self.buffer[: self.pointer]
+
+
+class BackpressureHandler:
+    # Track the number of messages currently being processed
+    current_parallel_executions = 0
+    # helps detect IDLE condition
+    last_data_message_time = 0
+    # helps prevent flooding the system with backpressure events
+    last_backpressure_event_time = 0
+    last_backpressure_event = ScalingRequestAlert.UPDATE
+    # _callback_list is used in unit tests to record the invoked callbacks
+    _callback_list = None
+
+    def __init__(
+        self,
+        channel: AbstractRobustChannel,
+        loop: AbstractEventLoop,
+        config: AMQConfiguration,
+    ):
+        self.lock = None
+        self.channel = channel
+        self.loop = loop
+        self.config = config
+        self.exchange = None
+        self.swarm_service_id = self.config.amq_adapter.swarm_service_id
+        self.swarm_task_id = self.config.amq_adapter.swarm_task_id
+        self.parallel_workers = self.config.backpressure.threshold_threads
+        self.average_cpu_usage = None
+        self.do_loop = -1
+
+    def _do_loop(self) -> bool:
+        """
+        Check if the loop should continue running.
+        Positive value of do_loop indicates the number of iterations left.
+        Negative value indicates the loop should run indefinitely.
+        """
+        _val = self.do_loop != 0
+        if self.do_loop > 0:
+            self.do_loop -= 1
+        return _val
+
+    def increase_parallel_executions(self):
+        """Increase the number of parallel executions"""
+        logging_info(
+            "Backpressure: Increase parallel executions, current=%s",
+            self.current_parallel_executions,
+        )
+        self.current_parallel_executions += 1
+
+    def decrease_parallel_executions(self):
+        """Decrease the number of parallel executions"""
+        logging_info(
+            "Backpressure: Decrease parallel executions, current=%s",
+            self.current_parallel_executions,
+        )
+        if self.current_parallel_executions > 0:
+            self.current_parallel_executions -= 1
+
+    def update_parallel_executions(self, count: int):
+        """Update the number of parallel executions"""
+        self.current_parallel_executions = count
+
+    def update_last_data_message_time(self):
+        logging_info("Backpressure: Update last data message time")
+        """Update the last data message time"""
+        self.last_data_message_time = time.time()
+
+    def start_backpressure_monitor(self) -> Thread:
+        # Start the Backpressure monitor loop
+        self.thread = Thread(target=self.backpressure_monitor_loop)
+        self.thread.daemon = True  # This makes it a daemon thread
+        self.thread.start()
+        return self.thread
+
+    async def check_overload_condition(self):
+        """Check if the current parallel executions exceed the limit"""
+        self.update_last_data_message_time()
+        logging_info(
+            "Backpressure: Check overload condition, current=%s, max=%s",
+            self.current_parallel_executions,
+            self.parallel_workers,
+        )
+        if self.current_parallel_executions >= self.parallel_workers - 1:
+            # Check if the last backpressure event was not an overload
+            if self.last_backpressure_event != ScalingRequestAlert.OVERLOAD:
+                # Trigger the overload event
+                await self.handle_backpressure_overload_event()
+                self.last_backpressure_event_time = time.time()
+                self.last_backpressure_event = ScalingRequestAlert.OVERLOAD
+
+    async def _backpressure_monitor(self):
+        """Monitor the backpressure conditions and trigger events accordingly"""
+        _time_window_millis = self.config.backpressure.time_window
+        _idle_duration_millis = self.config.backpressure.idle_duration
+        _overload_duration_millis = self.config.backpressure.cpu_overload_duration
+        _monitor_interval = 0.5  # Monitor every 500ms
+        CPU_IDLE, CPU_ACTIVE, CPU_OVERLOAD = 0, 1, 2
+        IDLE_THRESHOLD, OVERLOAD_THRESHOLD = 15, 90
+        if self.average_cpu_usage is None:
+            self.average_cpu_usage = RunningAverage(
+                int(max(_overload_duration_millis, _idle_duration_millis) // _monitor_interval)
+            )
+        _cpu_usage_state = CPU_ACTIVE
+        _cpu_usage_changed = time.time()
+        while self._do_loop():
+            _current_cpu_usage = psutil.cpu_percent(interval=None)
+            self.average_cpu_usage.add(_current_cpu_usage)
+            _average_cpu_usage = self.average_cpu_usage.get_average()
+            _now = time.time()
+            # logging_info(
+            #    "Backpressure: current CPU[pct]=%s, avg=%s, state=%s, last dataMsg=%s, last event=%s, last eventTime=%s",
+            #    _current_cpu_usage,
+            #    _average_cpu_usage,
+            #    _cpu_usage_state,
+            #    self.last_data_message_time,
+            #    self.last_backpressure_event,
+            #    self.last_backpressure_event_time,
+            # )
+            if _average_cpu_usage < IDLE_THRESHOLD:
+                if _cpu_usage_state != CPU_IDLE:
+                    _cpu_usage_changed = _now
+                _cpu_usage_state = CPU_IDLE
+            elif _average_cpu_usage > OVERLOAD_THRESHOLD:
+                if _cpu_usage_state != CPU_OVERLOAD:
+                    _cpu_usage_changed = _now
+                _cpu_usage_state = CPU_OVERLOAD
+            else:
+                if _cpu_usage_state != CPU_ACTIVE:
+                    _cpu_usage_changed = _now
+                _cpu_usage_state = CPU_ACTIVE
+
+            # Check if the current time exceeds the last backpressure event time
+            if (
+                _cpu_usage_state == CPU_OVERLOAD
+                and _now - _cpu_usage_changed > _overload_duration_millis
+                and (
+                    _now - self.last_backpressure_event_time > _time_window_millis
+                    or self.last_backpressure_event != ScalingRequestAlert.OVERLOAD
+                )
+            ):
+                # Trigger the overload event
+                await self.handle_backpressure_overload_event()
+                self.last_backpressure_event_time = _now
+                self.last_backpressure_event = ScalingRequestAlert.OVERLOAD
+            elif (
+                _cpu_usage_state == CPU_IDLE
+                and _now - _cpu_usage_changed > _idle_duration_millis
+                and _now - self.last_data_message_time > _idle_duration_millis
+                and (
+                    _now - self.last_backpressure_event_time > _time_window_millis
+                    or self.last_backpressure_event != ScalingRequestAlert.IDLE
+                )
+            ):
+                # Trigger the idle event
+                await self._handle_backpressure_idle_event()
+                self.last_backpressure_event = ScalingRequestAlert.IDLE
+                self.last_backpressure_event_time = _now
+            else:
+                # Trigger update event
+                if _now - self.last_backpressure_event_time > _time_window_millis:
+                    # Trigger the update event
+                    _scaling_request: ScaleRequestV1 = ScaleRequestV1(
+                        self.swarm_service_id,
+                        self.swarm_task_id,
+                        100,
+                        _average_cpu_usage,
+                        ScalingRequestAlert.UPDATE,
+                    )
+                    # Address the message to any adapter capable of supporting BACKPRESSURE request
+                    await self.publish_backpressure_request(_scaling_request)
+                    self.last_backpressure_event_time = _now
+                    self.last_backpressure_event = ScalingRequestAlert.UPDATE
+            await asyncio.sleep(_monitor_interval)
+
+    def backpressure_monitor_loop(self):
+        _loop = asyncio.new_event_loop()
+        _loop.run_until_complete(self._backpressure_monitor())
+
+    async def handle_backpressure_overload_event(self):
+        logging_warning("Backpressure: Capacity close to depleted!")
+        # Send an Overload event to the Management service
+        scaling_request: ScaleRequestV1 = ScaleRequestV1(
+            self.swarm_service_id,
+            self.swarm_task_id,
+            self.parallel_workers,
+            self.parallel_workers - self.current_parallel_executions,
+            ScalingRequestAlert.OVERLOAD,
+        )
+        # Address the message to any management service instance capable of supporting BACKPRESSURE request
+        await self.publish_backpressure_request(scaling_request)
+        # update / reset time-window so that the OVERLOAD is not sent too often
+        self.last_data_message_time = time.time()
+
+    async def _handle_backpressure_idle_event(self):
+        logging_warning("Backpressure: Service is idle.")
+        # Send an Idle event to the Management service
+        scaling_request: ScaleRequestV1 = ScaleRequestV1(
+            self.swarm_service_id,
+            self.swarm_task_id,
+            self.parallel_workers,
+            self.parallel_workers - self.current_parallel_executions,
+            ScalingRequestAlert.IDLE,
+        )
+        # Address the message to any adapter capable of supporting BACKPRESSURE request
+        await self.publish_backpressure_request(scaling_request)
+        # update / reset time-window so that the IDLE is not sent too often
+        self.last_data_message_time = time.time()
+
+    async def publish_backpressure_request(self, scaling_request: ScaleRequestV1):
+        # Publish the backpressure request to the management service
+        logging_info(
+            f"Publishing backpressure for {scaling_request.serviceId} with request type {scaling_request.requestType}"
+        )
+        if not self.exchange:
+
+            async def _wrap_rabbit_mq_api_init(channel):
+                _exchange = await channel.get_exchange(name="cleverthis.clevermicro.management")
+                return _exchange
+
+            if BackpressureHandler._callback_list is not None:
+                BackpressureHandler._callback_list["_wrap_rabbit_mq_api_init"] = 1
+            self.exchange = await await_result(
+                asyncio.run_coroutine_threadsafe(_wrap_rabbit_mq_api_init(self.channel), self.loop)
+            )
+
+        if self.exchange:
+
+            async def _wrap_rabbit_mq_api():
+                if not self.channel.is_closed:
+                    binary_content: bytes = scaling_request.to_json().encode("utf-8")
+                    pika_message: Message = Message(
+                        body=binary_content,
+                        content_encoding="utf-8",
+                        delivery_mode=2,
+                        content_type="application/octet-stream",
+                        headers=None,
+                        priority=0,
+                        correlation_id=None,
+                    )
+                    await self.exchange.publish(
+                        message=pika_message, routing_key="backpressure-scaling-v1"
+                    )
+                    logging_info(
+                        "Service Message Published to %s, msg: %s",
+                        self.exchange.name,
+                        str(binary_content),
+                    )
+                    return True
+                return False
+
+            if BackpressureHandler._callback_list is not None:
+                BackpressureHandler._callback_list["_wrap_rabbit_mq_api"] = 1
+            await await_future(asyncio.run_coroutine_threadsafe(_wrap_rabbit_mq_api(), self.loop))

amqp/adapter/cleverthis_service_adapter.py

@@ -0,0 +1,781 @@
+"""
+The layer that interfaces with the CleverThis service that this Adapter is integrating with.
+Methods here will invoke the actual CleverThis Service code, which are passed as Callable object.
+"""
+
+import asyncio
+import enum
+import inspect
+import json
+import pathlib
+import re
+from asyncio import AbstractEventLoop
+from collections import defaultdict
+from typing import Any, Callable, Coroutine, Dict, List
+
+from aio_pika.abc import (
+    AbstractExchange,
+    AbstractRobustChannel,
+    AbstractRobustConnection,
+)
+from aiohttp import ClientResponse, ClientSession
+from fastapi import HTTPException, UploadFile
+from fastapi.params import File
+from fastapi.routing import APIRoute
+from fastapi.security import OAuth2PasswordRequestForm
+from pydantic import BaseModel
+from starlette.datastructures import Headers
+from starlette.responses import FileResponse
+
+from amqp.adapter import serializer
+from amqp.adapter.data_parser import AMQDataParser
+from amqp.adapter.data_response_factory import DataResponseFactory
+from amqp.adapter.file_handler import FileHandler, StreamingFileHandler
+from amqp.adapter.logging_utils import (
+    logging_debug,
+    logging_error,
+    logging_info,
+    logging_warning,
+)
+from amqp.adapter.pydantic_serializer import python_type_to_json
+from amqp.adapter.type_descriptor import TypeDescriptor
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import AMQErrorMessage, AMQMessage, DataMessage, DataResponse
+from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
+from amqp.router.utils import await_result
+
+g_extra_init_data: dict = dict()
+
+
+async def _create_reply_exchange_and_queue(
+    connection: AbstractRobustConnection,
+) -> tuple[AbstractRobustChannel, AbstractExchange, str]:
+    """
+    Create objects for reply.
+    Return (channel for reply queue, exchange for reply, reply queue name)
+    """
+
+    # Wrap calls to RabbitMQ API to make it Awaitable, so it can be run thread-safe with
+    # correct event loop. This is needed because the RabbitMQ API is not thread-safe and needs
+    # to be called in the context of event loop the first RabbitMQ connection was created with.
+    _channel = await connection.channel()
+    _exchange_name: str = AMQ_REPLY_TO_EXCHANGE
+    _exchange = await _channel.get_exchange(name=_exchange_name, ensure=True)
+    _queue = await _channel.declare_queue(exclusive=False)  # Declare a unique, non-exclusive queue
+    _queue_name = _queue.name
+    await _queue.bind(
+        exchange=_exchange_name,
+        routing_key=_queue_name,  # Use queue name as routing key
+    )
+    logging_debug(f"Publishing to exchange: {_exchange_name}, q: {_queue_name}")
+    return _channel, _exchange, _queue_name
+
+
+async def _convert_file_response(
+    obj: FileResponse,
+    connection: AbstractRobustConnection,
+    file_handler: FileHandler,
+    loop: AbstractEventLoop | None,
+) -> str:
+    """
+    Converts a FileResponse object to a JSON string containing the filename and stream name.
+    The file content itself is transmitted through the named stream (which is implemented as
+    temporary dedicated queue.
+    """
+    # All calls to RabbitMQ API have to be called threadsafe, because those appear to be attached
+    # to parent event loop, and need to execute at that event loop, not the current one
+    # allow coroutine to execute. calling _future.result() will block the entire thread
+    # and the coroutine will never execute
+    _channel, _exchange, _routing_key = await await_result(
+        asyncio.run_coroutine_threadsafe(_create_reply_exchange_and_queue(connection), loop)
+    )
+    # publish_file also needs RabbitMQ API, ensure the loop is passed as argument
+    (_stream_name, _file_size) = await file_handler.publish_file(
+        _exchange, obj.path, _routing_key, loop
+    )
+    # as above, but luckily this time we don't need to wait for the result
+    asyncio.run_coroutine_threadsafe(_channel.close(), loop)
+    return json.dumps(
+        {
+            "files": [
+                {
+                    "filename": obj.filename,
+                    "mediaType": "application/octet-stream",
+                    "streamName": _stream_name,
+                    "fileSize": _file_size,
+                }
+            ]
+        }
+    )
+
+
+def is_likely_json(text: str) -> bool:
+    """Quick test to see if should convert the response string to JSON format."""
+    if not isinstance(text, str):
+        return False
+    _text = text.strip()
+    return (_text.startswith("{") and _text.endswith("}")) or (
+        _text.startswith("[") and _text.endswith("]")
+    )
+
+
+# =============  CleverThisServiceAdapter =============
+class CleverThisServiceAdapter:
+    """
+    A base class for CleverThis service adapter that handles the incoming AMQP messages and routes
+    them to the appropriate CleverThisService API endpoint.
+    IMPORTANT: This class is not intended to be used directly. It should be subclassed for
+    each CleverThis service and override the methods to handle the specific API calls.
+    """
+
+    def __init__(
+        self,
+        amq_configuration: AMQConfiguration,
+        routes: List[APIRoute],
+        session: ClientSession = None,
+        authenticated_user_arg_name: str = "authenticated_user",
+        extra_init_data=None,
+    ):
+        """
+        Initializes the CleverThisServiceAdapter with the given AMQ configuration and optional HTTP
+        session. The HTTP session is for the case when the adapter uses loose coupling with
+        the CleverThis service via HTTP REST API.
+        """
+        if extra_init_data is None:
+            extra_init_data = dict()
+        self.authenticated_user_arg_name = authenticated_user_arg_name
+        self.amq_configuration = amq_configuration
+        self.session = session
+        self.service_host = self.amq_configuration.dispatch.service_host
+        self.service_port = self.amq_configuration.dispatch.service_port
+        self.loop: AbstractEventLoop | None = None
+        self.require_authenticated_user = (
+            self.amq_configuration.amq_adapter.require_authenticated_user
+        )
+        self.routes = self.group_routes_by_method(api_routes=routes)
+        global g_extra_init_data
+        g_extra_init_data = extra_init_data or {}
+
+    def get_endpoint(self, endpoint: Any) -> Any:
+        """
+        Optionally, Service can override this method to alter the endpoint mapping.
+        This is useful for example when the CleverThis service has a JSON wrapper
+        around the endpoint, like CleverSwarm, which might be the preferred endpoint to call.
+
+        :param endpoint: The original endpoint to be called.
+        :return: The actual endpoint to be called.
+        """
+        return endpoint
+
+    def clone_and_adapt_route(self, original_route: APIRoute) -> APIRoute:
+        """
+        Creates a clone of APIRoute. Checks if there's preferred endpoint with json serialization.
+        It also adjusts the path regex to ensure it matches the path in different API versions.
+        """
+
+        # Need to adapt also the path regex to ensure it matches the path in different API versions
+        # Ensures the leading '^' is followed by any match and there's no '/' before final '$'
+        _new_regex_string = "^.*" + original_route.path_regex.pattern.lstrip("^").rstrip("/$") + "$"
+        _new_route: APIRoute = APIRoute(
+            path=original_route.path,
+            endpoint=self.get_endpoint(
+                original_route.endpoint
+            ),  # Service can alter the endpoint mapping here
+            response_model=original_route.response_model,
+            status_code=original_route.status_code,
+            tags=original_route.tags.copy() if original_route.tags else None,
+            dependencies=(
+                original_route.dependencies.copy() if original_route.dependencies else None
+            ),
+            summary=original_route.summary,
+            description=original_route.description,
+            response_description=original_route.response_description,
+            responses=(original_route.responses.copy() if original_route.responses else None),
+            deprecated=original_route.deprecated,
+            name=original_route.name,
+            methods=original_route.methods.copy() if original_route.methods else None,
+            operation_id=original_route.operation_id,
+            response_model_include=original_route.response_model_include,
+            response_model_exclude=original_route.response_model_exclude,
+            response_model_by_alias=original_route.response_model_by_alias,
+            response_model_exclude_unset=original_route.response_model_exclude_unset,
+            response_model_exclude_defaults=original_route.response_model_exclude_defaults,
+            response_model_exclude_none=original_route.response_model_exclude_none,
+            include_in_schema=original_route.include_in_schema,
+            response_class=original_route.response_class,
+            dependency_overrides_provider=original_route.dependency_overrides_provider,
+            callbacks=(original_route.callbacks.copy() if original_route.callbacks else None),
+            openapi_extra=(
+                original_route.openapi_extra.copy() if original_route.openapi_extra else None
+            ),
+            generate_unique_id_function=original_route.generate_unique_id_function,
+        )
+        _new_route.path_regex = re.compile(_new_regex_string)
+        return _new_route
+
+    def group_routes_by_method(self, api_routes: List[APIRoute]) -> Dict[str, List[APIRoute]]:
+        """
+        Groups API routes by their HTTP method. This is important to ensure that URLPath is matched
+        correctly, as different methods may share the same URLPath (e.g. GET, POST PUT, DELETE).
+        """
+        _method_to_routes = defaultdict(list)  # Automatically initializes new lists for new keys
+        for _route in api_routes:
+            if (
+                isinstance(_route, APIRoute) and _route.methods
+            ):  # Ensure methods exist (should always be true for valid routes)
+                for method in _route.methods:
+                    _method_to_routes[method].append(self.clone_and_adapt_route(_route))
+
+        return dict(_method_to_routes)
+
+    def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
+        """
+        Pull the value of the Content-Type header from the message headers.
+        :param message_headers: The headers of the message.
+        return: The content type of the message.
+        """
+        for header, values in message_headers.items():
+            if header.lower() == "content-type":
+                return values[0].split(";")[0]
+        return "application/json"
+
+    async def get_current_user(self, token: str) -> object:
+        """
+        To Be Overriden in subclass for given CleverThis Service, to return the current user based
+        on the token, in format appropriate for the service.
+
+        :param token: The token to be used for authentication.
+
+        :return: A user object or None if the user is not authenticated.
+        """
+        return None
+
+    async def on_message(self, message: AMQMessage) -> DataResponse:
+        """
+        Called when a message is received from the AMQP.
+        :param message:
+        :return: AMQP response class with operation status code and optional error details.
+        """
+        _auth_user: Any | None = None
+        _amq_response: Any | None = None
+
+        logging_debug(f"on_message: require_authenticated_user={self.require_authenticated_user}")
+        if self.require_authenticated_user:
+            _auth: str = message.headers().get("Authorization", "")
+            logging_debug(f"Auth: {_auth}")
+            if isinstance(_auth, List):
+                _auth = _auth[0]
+            if "Bearer" in _auth:
+                try:
+                    token = _auth.split(" ")[1]
+                    logging_info(f"Token: {token}")
+                    _auth_user = await self.get_current_user(token)
+                except Exception as error:
+                    logging_error(f"on_message: Error getting user: {error}")
+
+        if _auth_user or not self.require_authenticated_user or message.path().endswith("/login"):
+            method = message.method().lower()
+            if method == "get":
+                _amq_response = await self.get(message, _auth_user)
+            elif method == "put":
+                _amq_response = await self.put(message, _auth_user)
+            elif method == "post":
+                _amq_response = await self.post(message, _auth_user)
+            elif method == "head":
+                _amq_response = await self.head(message, _auth_user)
+            elif method == "delete":
+                _amq_response = await self.delete(message, _auth_user)
+            else:
+                raise ValueError(f"Unexpected HTTP method: {message.method()}")
+        else:
+            _amq_response = DataResponseFactory.of_error_message(
+                message.id(),
+                AMQErrorMessage(401, "Unauthorized", "Unauthorized"),
+                message.trace_info(),
+            )
+        logging_info(
+            f"ALL DONE in on_message id:{message.id()}, resp code:{_amq_response.response_code()}"
+        )
+        return _amq_response
+
+    async def _authorize_forward_request(
+        self, message: DataMessage, request_coroutine, auth_user: Any
+    ) -> DataResponse:
+        """
+        This method is used to authorize the request and forward it to the CleverThis Service API.
+        Used only for loose / 3rd party mode of coupling with the Service
+        """
+        # AMQDataParser.parse_request_body(message)
+        # TODO: implement authorization logic here - ensure the security related headers are set
+        # This depends on what kind of security is required by the CleverThis service.
+        # However at this time there is no loosely coupled Service.
+
+        return await request_coroutine
+
+    # ==============================================================================================
+    # =====================  C l e v e r T h i s    A P I    m e t h o d s  ========================
+    # ==============================================================================================
+
+    async def get(self, message: AMQMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the GET requests for the CleverThis API. In tight coupling (self.session is None),
+        use path, path params and query string to determine the endpoint method to invoke.
+        In loose coupling (self.session is not None), forward the request as REST request using
+        the session, i.e. attempt to call the REST endpoint directly using HTTP protocol.
+
+        :param message: message from the AMQP that contains the GET request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+
+        if self.session is None:
+            return await self.handle_no_body_payload(message, auth_user, "GET")
+
+        # Forwards as REST - try to invoke the service directly on this path
+        url = f"http://{self.service_host}:{self.service_port}{message.path()}"
+        headers = {**message.headers(), **message.trace_info()}
+        logging_info(
+            f"SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}"
+        )
+        _http_resp: ClientResponse = await self.session.get(url, headers=headers)
+        return DataResponseFactory.of(
+            message.id(),
+            _http_resp.status,
+            _http_resp.content_type,
+            await _http_resp.read(),
+            message.trace_info(),
+        )
+
+    async def put(self, message: AMQMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the PUT requests for the CleverThis API. Uses path, path params, query string and
+        body payload to determine the endpoint method to invoke and its input arguments
+        In loose coupling (self.session is not None), attempt to call the REST endpoint directly
+        using HTTP protocol.
+
+        :param message: message from the AMQP that contains the PUT request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        if self.session is None:
+            return await self.handle_body_payload(message, auth_user, "PUT")
+
+        # Forwards as REST - try to invoke the service directly on this path
+        return await self._authorize_forward_request(
+            message,
+            self.session.put(
+                f"http://{self.service_host}:{self.service_port}{message.path()}",
+                data=message.body(),
+                headers={**message.headers(), **message.trace_info()},
+            ),
+            auth_user,
+        )
+
+    async def post(self, message: AMQMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the POST requests for the CleverThis API. Uses path, path params, query string and
+        body payload to determine the endpoint method to invoke and its input arguments
+        In loose coupling (self.session is not None), attempt to call the REST endpoint directly
+        using HTTP protocol.
+
+        :param message: message from AMQP that contains the POST request, including body and headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        if self.session is None:
+            return await self.handle_body_payload(message, auth_user, "POST")
+
+        return await self._authorize_forward_request(
+            message,
+            self.session.post(
+                f"http://{self.service_host}:{self.service_port}{message.path()}",
+                data=message.body(),
+                headers={**message.headers(), **message.trace_info()},
+            ),
+            auth_user,
+        )
+
+    async def head(self, message: AMQMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the HEAD requests for the CleverThis API. In tight coupling (self.session is None),
+        use path, path params and query string to determine the endpoint method to invoke.
+        In loose coupling (self.session is not None), HEAD call is not supported.
+
+        :param message: message from AMQP containing the HEAD request, including body and headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchema
+
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        if self.session is None:
+            return await self.handle_no_body_payload(message, auth_user, "HEAD")
+
+        raise NotImplementedError
+
+    async def delete(self, message: AMQMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the DELETE requests for the CleverThis API.
+
+        :param message: message from AMQP that contains the DELETE request, including body & headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchema
+
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        if self.session is None:
+            return await self.handle_no_body_payload(message, auth_user, "DELETE")
+
+        return await self._authorize_forward_request(
+            message,
+            self.session.post(
+                f"http://{self.service_host}:{self.service_port}{message.path()}",
+                data=message.body(),
+                headers={**message.headers(), **message.trace_info()},
+            ),
+            auth_user,
+        )
+
+    async def options(self, message: AMQMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the OPTIONS requests. In tight coupling (self.session is None),
+        use path, path params and query string to determine the endpoint method to invoke.
+        In loose coupling (self.session is not None), forward the request as REST request using
+        the session, i.e. attempt to call the REST endpoint directly using HTTP protocol.
+
+        :param message: message from AMQP that contains the OPTIONS request, including headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchema
+
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        if self.session is None:
+            return await self.handle_no_body_payload(message, auth_user, "OPTIONS")
+
+        raise NotImplementedError
+
+    async def patch(self, message: AMQMessage, auth_user: Any) -> DataResponse:
+        """
+        Handles the PATCH requests for the CleverBRAG API.
+
+        :param message: message from AMQP that contains the PATCH request, including body & headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchema
+
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        if self.session is None:
+            return await self.handle_no_body_payload(message, auth_user, "PATCH")
+
+        return await self._authorize_forward_request(
+            message,
+            self.session.post(
+                f"http://{self.service_host}:{self.service_port}{message.path()}",
+                data=message.body(),
+                headers={**message.headers(), **message.trace_info()},
+            ),
+            auth_user,
+        )
+
+    # ==============================================================================================
+    # =================  C l e v e r S w a r m   p a y l o a d   h a n d l e r s  ==================
+    # ==============================================================================================
+
+    async def handle_no_body_payload(
+        self, message: AMQMessage, auth_user: Any, method: str
+    ) -> DataResponse:
+        """
+        Handles the requests that do not have a body payload.
+        :param message: message from the AMQP that contains the request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        _routes = self.routes.get(method, [])
+        _path, _args = AMQDataParser.parse_get_url(message.path())
+        for _route in _routes:
+            _match = _route.path_regex.match(_path)
+            if _match:
+                _args.update(_match.groupdict())
+                _signature_args = inspect.signature(_route.endpoint).parameters
+                if self.authenticated_user_arg_name in _signature_args.keys():
+                    _args[self.authenticated_user_arg_name] = auth_user
+                # If the path matches, call the corresponding function
+                return await self.call_cleverthis_api(
+                    message,
+                    _args,
+                    api_method=_route.endpoint,
+                    success_code=_route.status_code,
+                )
+
+        return DataResponseFactory.of(
+            message.id(),
+            404,
+            "text/plain",
+            str(f"Destination location [{message.path()}] does not exists").encode("utf-8"),
+            message.trace_info(),
+        )
+
+    async def handle_body_payload(
+        self, message: AMQMessage, auth_user: Any, method: str
+    ) -> DataResponse:
+        """
+        Handles the requests that have a body payload, including Multipart data.
+        :param message: message from the AMQP that contains the request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :param method: HTTP method (POST, PUT, etc.)
+        :return: DataResponse to be sent back to the API Gateway via AMQP
+        """
+        _form_data = AMQDataParser.parse_request_body(message)
+        _routes = self.routes.get(method, [])
+        _path, _query_args = AMQDataParser.parse_get_url(message.path())
+        for _route in _routes:
+            _match = _route.path_regex.match(_path)
+            if _match:
+                # If the path matches, call the corresponding function
+                _form_data.update(_match.groupdict())
+                _form_data.update(_query_args)
+                for _body_param in _route.dependant.body_params:
+                    if _body_param.name in _form_data:
+                        _file_data = _form_data.get(_body_param.name, None)
+                        actual_filepath = message.extra_data.get(_body_param.name, None)
+                        _form_data[_body_param.name] = (
+                            (
+                                _body_param.type_.__name__ == "UploadFile"
+                                or isinstance(_body_param.field_info, File)
+                            )
+                            and _file_data is not None
+                            and actual_filepath is not None
+                            and UploadFile(
+                                file=pathlib.Path(actual_filepath).open("rb"),
+                                size=_file_data.get("size"),
+                                filename=_file_data.get("filename"),
+                                headers=Headers({"Content-Type": _file_data.get("mediaType")}),
+                            )
+                            or _form_data[_body_param.name]
+                        )
+
+                _verified_args = {}
+                _required_args = getattr(_route.endpoint, "__annotations__", {})
+                _signature_args = inspect.signature(_route.endpoint).parameters
+                for arg in _required_args:
+                    if arg in _form_data:
+                        _verified_args[arg] = _form_data[arg]
+                    else:
+                        # needed for CleverSwarm /login  input
+                        if _required_args[arg].__name__ == "OAuth2PasswordRequestForm":
+                            try:
+                                _verified_args[arg] = OAuth2PasswordRequestForm(**_form_data)
+                            except Exception as e:
+                                print(f"Error: {e}")
+                if self.authenticated_user_arg_name in _signature_args.keys():
+                    _verified_args[self.authenticated_user_arg_name] = auth_user
+
+                return await self.call_cleverthis_api(
+                    message,
+                    _verified_args,
+                    api_method=_route.endpoint,
+                    success_code=_route.status_code if _route.status_code else 200,
+                )
+
+        return DataResponseFactory.of(
+            message.id(),
+            404,
+            "text/plain",
+            str(f"Destination location [{message.path()}] does not exists").encode("utf-8"),
+            message.trace_info(),
+        )
+
+    def _verify_and_instantiate_arguments(
+        self,
+        args: Any,
+        api_method: Callable[[Any, Any], Coroutine],
+    ) -> dict:
+        """
+        Check the provided arguments against the API method signature and convert the incoming string value
+        into correct expected input object.
+        The logic used FastAPI annotation and would use the value from 'default_factory' invocation (where defined)
+        or 'default' value, as defined in the FastAPI annotation, to create default value in case the object is not in
+        the input values provided by the caller.
+
+        """
+        _verified_args = {}
+        try:
+            _sig = inspect.signature(api_method)
+            _required_args = list(_sig.parameters.values())
+            # _required_args = getattr(api_method, '__annotations__', {})
+            for arg in _required_args:
+                if arg.name in args:
+                    _annotation = arg.annotation
+                    _str_annotation = str(_annotation)
+                    # convert to another type if arg is not a str
+                    if arg.name == self.authenticated_user_arg_name:
+                        _verified_args[arg.name] = args.get(arg.name)
+                    elif _annotation == int:
+                        _verified_args[arg.name] = int(args.get(arg.name))
+                    elif _annotation == float:
+                        _verified_args[arg.name] = float(args.get(arg.name))
+                    elif _annotation == bool:
+                        _verified_args[arg.name] = serializer.str_to_bool(args.get(arg.name))
+                    elif (
+                        "Optional" in _str_annotation
+                        or _str_annotation.lower().startswith("list")
+                        or _str_annotation.startswith("typing.List")
+                        or "class" in _str_annotation
+                    ):
+                        _type_descr = TypeDescriptor(
+                            _str_annotation, extra_init_data=g_extra_init_data
+                        )
+                        try:
+                            _payload = args.get(arg.name, "")
+                            _verified_args[arg.name] = _type_descr.instantiate(
+                                _payload, is_json=is_likely_json(_payload)
+                            )
+                        except Exception as err:
+                            logging_error(
+                                f"Could not instantiate argument {arg}, defaulting to its string value",
+                                err,
+                            )
+                            _verified_args[arg.name] = args.get(arg.name, None)
+                    else:
+                        _temp = args.get(arg.name)
+                        if isinstance(arg.annotation, enum.EnumType) or isinstance(
+                            arg.annotation, enum.Enum
+                        ):
+                            try:
+                                _temp = arg.annotation[_temp]
+                            except KeyError as ke:
+                                # some enums have string value different to key spelling, try different thing
+                                try:
+                                    _temp = arg.annotation(_temp)
+                                except ValueError as ve:
+                                    logging_warning(
+                                        f"cannot convert '{_temp}' to Enum type {arg.annotation}, {ke}, {ve}"
+                                    )
+                            print(f"ENUM -> {_temp}")
+                        _verified_args[arg.name] = _temp
+                else:
+                    if hasattr(arg, "default"):
+                        _temp = arg.default
+                        if hasattr(_temp, "default_factory") and _temp.default_factory is not None:
+                            _temp = _temp.default_factory()
+                        elif hasattr(_temp, "default"):
+                            _temp = _temp.default
+                        if not str(_temp) == "<class 'inspect._empty'>":
+                            _verified_args[arg.name] = _temp
+                    else:
+                        logging_warning(
+                            f"{api_method.__name__}: Missing required argument: {arg.name}"
+                        )
+        except Exception as e:
+            logging_error("Failed to validate and instantiate input args", e)
+
+        return _verified_args
+
+    async def call_cleverthis_api(
+        self,
+        message: AMQMessage,
+        args: Any,
+        api_method: Callable[[Any, Any], Coroutine],
+        success_code: int,
+    ) -> DataResponse:
+        """
+        Handles POST and PUT requests, which typically contain possibly complex payload data,
+        passed as a dictionary. Invokes the provided API method with the given arguments and
+        authenticated user, and returns the response as DataResponse.
+
+        :param message: DataMessage containing the details of the call and the payload
+        :param args: map of the arguments to pass to the API method
+        :param api_method: Callable - the method to invoke. Last parameter is the authenticated user
+        :param success_code: HTTP code to be returned when operation suceeds
+
+        :return: DataResponse class
+        """
+        try:
+            _verified_args = self._verify_and_instantiate_arguments(
+                args=args, api_method=api_method
+            )
+            logging_info(
+                "INVOKE ENDPOINT: [%s], ARGS: {%s}",
+                api_method.__name__,
+                ", ".join(f"{k}={v}" for k, v in _verified_args.items()),
+            )
+            _result: Any = await api_method(**_verified_args)
+            logging_info(f"ENDPOINT RESULT: [{api_method.__name__}], RESULT: {str(_result)}")
+
+            if isinstance(_result, FileResponse):
+                _json_result, _content_type = (
+                    await _convert_file_response(
+                        _result,
+                        message.connection,
+                        file_handler=StreamingFileHandler(),
+                        loop=self.loop,
+                    ),
+                    "application/octet-stream",
+                )
+            elif isinstance(_result, BaseModel):
+                _json_result = _result.json()
+                _content_type = "application/json"
+            else:
+                if isinstance(_result, str):
+                    _json_result = _result
+                else:
+                    _json_result = python_type_to_json(_result)
+                _content_type = "application/json"
+
+            _binary_result = (
+                _json_result.encode("utf-8")
+                if hasattr(_json_result, "encode")
+                else (
+                    _json_result.body
+                    if hasattr(_json_result, "body")
+                    else (_json_result.read() if hasattr(_json_result, "read") else _json_result)
+                )
+            )
+            logging_info(
+                f"ALL DONE in call_cleverthis_api id:{message.id()}, resp code:{success_code}"
+            )
+            return DataResponseFactory.of(
+                message.id(),
+                success_code,
+                _content_type,
+                _binary_result,
+                message.trace_info(),
+            )
+        except HTTPException as http_error:
+            logging_error(f"Service REST endpoint invocation failed: {http_error}")
+            _content = str(http_error.detail)
+            if not is_likely_json(_content):
+                return DataResponseFactory.of_error_message(
+                    message.id(),
+                    AMQErrorMessage(
+                        http_error.status_code,
+                        "Service Invocation Failed",
+                        _content,
+                    ),
+                    message.trace_info(),
+                )
+            return DataResponseFactory.of(
+                message.id(),
+                http_error.status_code,
+                "application/json",
+                _content.encode("utf-8"),
+                message.trace_info(),
+            )
+        except Exception as error:
+            logging_error(f"Service endpoint invocation failed: {error}")
+            _content = str(error)
+            if not is_likely_json(_content):
+                return DataResponseFactory.of_error_message(
+                    message.id(),
+                    AMQErrorMessage(
+                        500,
+                        "Service Invocation Failed",
+                        _content,
+                    ),
+                    message.trace_info(),
+                )
+            return DataResponseFactory.of(
+                message.id(),
+                500,
+                "application/json",
+                _content.encode("utf-8"),
+                message.trace_info(),
+            )

amqp/adapter/data_message_factory.py

@@ -0,0 +1,252 @@
+"""
+methods to create, serialize and deserialize DataMessage objects.
+"""
+
+import base64
+import json
+import re
+from datetime import datetime, timezone
+from io import BytesIO
+from typing import Dict, List
+
+import amqp.adapter.file_handler
+from amqp.adapter.file_handler import FileHandler, StreamingFileHandler
+from amqp.adapter.serializer import (
+    map_as_string,
+    map_with_list_as_string,
+    parse_map_list_string,
+    parse_map_string,
+)
+from amqp.model.model import DataMessage, DataMessageMagicByte
+from amqp.model.snowflake_id import SnowflakeId
+from amqp.router.utils import sanitize_as_rabbitmq_name
+
+
+class DataMessageFactory:
+    SNOWFLAKE_ID_BITS = 80
+    SEQUENCE_BITS = 12
+    MACHINE_ID_BITS = 24
+    TIMESTAMP_BITS = SNOWFLAKE_ID_BITS - SEQUENCE_BITS - MACHINE_ID_BITS
+
+    MACHINE_ID_MASK = (1 << MACHINE_ID_BITS) - 1
+    SEQUENCE_MASK = (1 << SEQUENCE_BITS) - 1
+
+    _instance = None
+    snowflake_epoch = datetime(2024, 10, 1, 0, 0, tzinfo=timezone.utc).timestamp()
+
+    def __init__(self, machine_id: int):
+        self.machine_id = (machine_id & self.MACHINE_ID_MASK) << self.SEQUENCE_BITS
+        self.snowflake_sequence = 0
+        self.snowflake_last_timestamp = 0
+
+    @classmethod
+    def get_instance(cls, machine_id: int):
+        """
+        return reusable factory instance
+        :param machine_id: a generator ID, should be unique for each component to prevent duplicate IDs
+        :return: factory instance
+        """
+        if cls._instance is None:
+            cls._instance = DataMessageFactory(machine_id)
+        return cls._instance
+
+    def generate_snowflake_id(self) -> SnowflakeId:
+        """
+        Generate an unique ID following the SnowflakeID structure, see https://en.wikipedia.org/wiki/Snowflake_ID
+        :return: 80-bit wide ID
+        """
+        timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
+
+        if timestamp < self.snowflake_last_timestamp:
+            raise RuntimeError("Clock moved backwards. Refusing to generate id")
+
+        if timestamp == self.snowflake_last_timestamp:
+            self.snowflake_sequence = (self.snowflake_sequence + 1) & self.SEQUENCE_MASK
+            if self.snowflake_sequence == 0:
+                while timestamp == self.snowflake_last_timestamp:
+                    timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
+                self.snowflake_last_timestamp = timestamp
+        else:
+            self.snowflake_sequence = 0
+            self.snowflake_last_timestamp = timestamp
+
+        time = timestamp - 1000 * int(DataMessageFactory.snowflake_epoch)
+        return SnowflakeId(
+            time >> (64 - self.SEQUENCE_BITS - self.MACHINE_ID_BITS),
+            (
+                (time << (self.SEQUENCE_BITS + self.MACHINE_ID_BITS))
+                | self.machine_id
+                | self.snowflake_sequence
+            )
+            & 0xFFFFFFFFFFFFFFFF,
+        )
+
+    def create_request_message(
+        self,
+        method: str,
+        domain: str,
+        path: str,
+        headers: Dict[str, List[str]],
+        message,
+    ) -> DataMessage:
+        """
+        Encapsulate / hide the system level inputs, create the AMQ message from business relevant input
+        :param method: a method requested (e.g. HTTP method like GET, POST in case of REST request)
+        :param domain: service host name
+        :param path: path at the service
+        :param headers: HTTP headers
+        :param message: payload
+        :return: DataMessage object
+        """
+        serializable_headers = headers.copy() if headers else {}
+        payload = base64.b64encode(message.encode("utf-8") if isinstance(message, str) else message)
+        trace_info = {}
+        return DataMessage(
+            DataMessageMagicByte.HTTP_REQUEST.value,
+            self.generate_snowflake_id(),
+            method,
+            domain,
+            path,
+            serializable_headers,
+            trace_info,
+            payload,
+        )
+
+    @staticmethod
+    def amq_message_routing_key(message: DataMessage) -> str:
+        """
+        Constructs the message specific routing key.
+        :param message: DataMessage input
+        :return: routing key (as string) compliant to RabbitMQ allowed character set.
+        """
+        return sanitize_as_rabbitmq_name(f"{message.domain()}.{message.path()}")
+
+    @staticmethod
+    def get_timestamp_from_id(_id: SnowflakeId) -> int:
+        """
+        Helper method - retrieve the timestamp from the ID
+        :param _id: SnowflakeID (includes encoded timestamp)
+        :return: the message timestamp
+        """
+        bits = _id.get_bits()
+        low = bits[0] >> (DataMessageFactory.SEQUENCE_BITS + DataMessageFactory.MACHINE_ID_BITS)
+        high = bits[1] << (
+            64 - DataMessageFactory.SEQUENCE_BITS - DataMessageFactory.MACHINE_ID_BITS
+        )
+        return (high | low) + DataMessageFactory.snowflake_epoch * 1000
+
+    @staticmethod
+    def to_string(message: DataMessage) -> str:
+        """
+        Human-readable representation.
+        :param message: DataMessage input
+        :return: as string
+        """
+        return (
+            f"DataMessage{{"
+            f"id={message.id()}, "
+            f"method='{message.method()}', "
+            f"domain='{message.domain()}', "
+            f"path='{message.path()}', "
+            f"headers={{{map_with_list_as_string(message.headers())}}}, "
+            f"traceInfo={{{map_as_string(message.trace_info())}}}, "
+            f"base64body={message.base64body().decode()}"
+            f"}}"
+        )
+
+    @staticmethod
+    def serialize(message: DataMessage) -> bytes:
+        """
+        Converts DataMessage to byte array
+        :param message: message to serialize
+        :return: messages as bytes
+        """
+        id_bytes = str(message.id()).encode()
+        prolog = (
+            f"|m={message.method()}|d={message.domain()}|p={message.path()}|h={{{map_with_list_as_string(message.headers())}}}|t={{{map_as_string(message.trace_info())}}}|b="
+        ).encode("utf-8")
+        header_length = 1 + len(id_bytes) + len(prolog)
+        prolog_len = header_length.to_bytes(2, byteorder="big")
+        msg_length = 2 + header_length + len(message.base64body())
+        with BytesIO(bytearray(msg_length)) as baos:
+            baos.write(prolog_len)
+            baos.write(DataMessageMagicByte.HTTP_REQUEST.value.encode("utf-8"))
+            baos.write(id_bytes)
+            baos.write(prolog)
+            baos.write(message.base64body())
+            return baos.getvalue()
+
+    @staticmethod
+    def from_bytes(input_bytes: bytes) -> DataMessage:
+        """
+        Builds the DataMessage from its serialized (byte array) form.
+        :param input_bytes: serialized message
+        :return: DataMessage instance
+        """
+        prolog_len = (input_bytes[0] << 8) + input_bytes[1]
+        metadata = input_bytes[2 : prolog_len + 2].decode()
+
+        pattern = re.compile(
+            f"{DataMessageMagicByte.HTTP_REQUEST.value}"
+            r"([0-9A-Fa-f]+)\|"
+            r"m=([^|]*)\|"
+            r"d=([^|]*)\|"
+            r"p=([^|]*)\|"
+            r"h=(\{[^}]*\})\|"
+            r"t=(\{[^}]*\})\|"
+            r"b="
+        )
+        match = pattern.match(metadata)
+        if not match:
+            raise ValueError(
+                f"DataMessage format does not match expected format. "
+                f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
+            )
+
+        id_str = match.group(1)
+        method = match.group(2)
+        domain = match.group(3)
+        path = match.group(4)
+        headers_str = match.group(5)
+        trace_info_str = match.group(6)
+        body_b64 = input_bytes[prolog_len + 2 :]
+
+        headers = parse_map_list_string(headers_str)
+        trace_info = parse_map_string(trace_info_str)
+
+        return DataMessage(
+            DataMessageMagicByte.HTTP_REQUEST.value,
+            SnowflakeId.from_hex(id_str),
+            method,
+            domain,
+            path,
+            headers,
+            trace_info,
+            body_b64,
+        )
+
+    @staticmethod
+    async def from_stream(
+        stream: bytes,
+        rabbitmq_url: str,
+        loop,
+        file_handler: FileHandler = StreamingFileHandler(),
+    ) -> DataMessage | None:
+        """
+        Builds the DataMessage from its serialized (byte array) form.
+        :param stream: serialized message
+        :param rabbitmq_url: RabbitMQ connection URL
+        :param loop: event loop
+        :param file_handler: file handler for downloading the message
+        :return: DataMessage instance
+        """
+        # async implementation
+        _message_data = json.loads(stream.decode())
+        _req_info = _message_data.get("req-info", "")
+        if _req_info:
+            (body, eof) = await file_handler.download_buffer(
+                loop=loop, rabbitmq_url=rabbitmq_url, queue_name=_req_info
+            )
+            if eof == amqp.adapter.file_handler.END_OF_FILE:
+                return DataMessageFactory.from_bytes(body)
+        return None

amqp/adapter/data_parser.py

@@ -0,0 +1,194 @@
+import io
+import json
+from typing import Dict, Tuple
+from urllib.parse import parse_qs, urlparse
+from xml.etree import ElementTree
+
+import python_multipart
+from fastapi import UploadFile
+from python_multipart.multipart import Field, File
+
+from amqp.adapter.logging_utils import logging_debug, logging_error
+from amqp.model.model import AMQMessage
+
+
+class MultipartFormDataParser:
+    """
+    Parses multipart/form-data content and extracts form fields (key/value pairs) and files.
+    Note: multipart message that contains file(s) to upload are converted to CleverMicro JSON-based streaming format,
+    where files are transmitted separately via stream using dedicated queue, so the message content is no longer
+    the HTTP multipart format, but the Content-Type is being preserved for compatibility with the original message.
+    This class supports both formats (multipart and JSON) depending on the message content, because if no file is
+    included, the message is transmitted in the unchanged multipart/form-data format.
+    """
+
+    def __init__(self, message: AMQMessage):
+        self.form_data: dict = {}
+        self.is_multipart = False
+        self.is_json = False
+        self.is_valid = True
+        # Convert each list of strings to a single string joined by newline, then to bytes
+        _headers = {
+            key: "\n".join(value).encode("utf-8") for key, value in message.headers().items()
+        }
+        try:
+            if (
+                hasattr(message, "extra_data")
+                and isinstance(message.extra_data, dict)
+                and len(message.extra_data) > 0
+            ):
+                try:
+                    self.form_data = json.loads(message.body())
+                    self.is_json = True
+                    if self.form_data["files"] is not None and self.form_data["form"] is not None:
+                        self.form_data = AMQDataParser.process_form_data(self.form_data)
+                except Exception as jsonerror:
+                    logging_error(f"Parsing JSON: {jsonerror}")
+                    self.is_valid = False
+            else:
+                python_multipart.parse_form(
+                    _headers, io.BytesIO(message.body()), self.on_field, self.on_file
+                )
+                self.is_multipart = True
+        except Exception as e:
+            __error = f"Error parsing multipart/form-data: {e}. Trying parsing as JSON."
+            try:
+                self.form_data = json.loads(message.body())
+                self.is_json = True
+                if self.form_data["files"] is not None and self.form_data["form"] is not None:
+                    self.form_data = AMQDataParser.process_form_data(self.form_data)
+            except Exception as jsonerror:
+                logging_error(__error)
+                logging_error(f"Parsing JSON: {jsonerror}")
+                self.is_valid = False
+
+    def on_field(self, field: Field):
+        logging_debug(str(field))
+        self.form_data[field.field_name.decode("utf-8")] = field.value
+
+    def on_file(self, file: File):
+        logging_debug(str(file))
+        file.file_object.seek(0)
+        self.form_data[file.field_name.decode("utf-8")] = UploadFile(
+            file.file_object, size=file.size, filename=file.file_name.decode("utf-8")
+        )
+
+
+class AMQDataParser:
+    """
+    A class for parsing AMQ data messages. It provides methods to parse form data,
+    """
+
+    @staticmethod
+    def process_form_data(json_input) -> dict:
+        """
+        Processes the input JSON to extract form data. The JSON has two top entries: 'files' and 'form'.
+        This is part of the file stream implementation in AMQ adapter to stream large files in chunks.
+        'files' contains information about the AMQ queue from which read the file, while 'form' contains
+        additional non-file form values.
+        This function combines the file and values at the top level in the returned dictionary.
+        Args:
+            json_input: JSON string or dict containing the form and files data
+
+        Returns:
+            dict: Processed form data with files data overriding non-empty values
+        """
+        # Load JSON if input is string
+        if isinstance(json_input, str):
+            data = json.loads(json_input)
+        else:
+            data = json_input
+
+        result = {}
+
+        # Process form data - take first element of each list
+        for key, value in data["form"].items():
+            if value:  # Only process if list is not empty
+                result[key] = value[0]
+
+        # Override with files data where available
+        for key, value in data["files"].items():
+            if value:  # Only override if files data is non-empty
+                result[key] = value[0]
+        return result
+
+    @staticmethod
+    def parse_get_url(url: str) -> Tuple[str, Dict[str, str]]:
+        """
+        Parses an HTTP GET URL and returns (context_path, query_params_dict)
+
+        Args:
+            url: The URL string to parse (e.g., "http://example.com/path?foo=1&bar=2&foo=3")
+
+        Returns:
+            tuple: (context_path, query_params_dict)
+            - context_path: The path part of the URL (e.g., "/path")
+            - query_params_dict: Query parameters where values are str for single value or list of str for multivalue
+        """
+        _parsed = urlparse(url)
+        _query_params = parse_qs(_parsed.query)
+        # Convert values from lists to single values when there's only one value
+        _simplified_params = {k: v[0] if len(v) == 1 else v for k, v in _query_params.items()}
+        return _parsed.path, _simplified_params
+
+    @staticmethod
+    def parse_request_body(message: AMQMessage):
+        """
+        Parses the HTTP POST request body based on the MIME type.
+        This is typically called fom the concrete CleverThis Service to extract parameters from the message body.
+        Args:
+            message (DataMessage): Contains raw request and MIME headers.
+
+        Returns:
+            dict: Parsed data as a dictionary.
+        """
+        _content_type = message.headers().get("Content-Type", "")
+        if not _content_type:
+            raise ValueError("Content-Type header is required")
+        content_type = _content_type[0]
+        # Parse JSON
+        if content_type == "application/json":
+            try:
+                return json.loads(message.body() or "{}")
+            except json.JSONDecodeError as e:
+                logging_error(f"Invalid JSON: {e}")
+                raise ValueError(f"Invalid JSON: {e}")
+
+        # Parse URL-encoded form data
+        elif content_type == "application/x-www-form-urlencoded":
+            try:
+                _form_data: dict = {
+                    k: v[0] if len(v) == 1 else v for k, v in parse_qs(message.body()).items()
+                }
+                if len(_form_data) == 0 and len(message.body()) > 0:
+                    _form_data = json.loads(message.body())
+                    if _form_data["files"] is not None and _form_data["form"] is not None:
+                        _form_data = AMQDataParser.process_form_data(_form_data)
+                return _form_data
+            except Exception as e:
+                logging_error(f"Invalid URL-encoded form data: {e}")
+                raise ValueError(f"Invalid URL-encoded form data: {e}")
+
+        # Parse multipart/form-data
+        elif content_type.startswith("multipart/form-data"):
+            try:
+                parser: MultipartFormDataParser = MultipartFormDataParser(message)
+                return parser.form_data
+            except Exception as e:
+                logging_error(f"Invalid multipart/form-data: {e}")
+                raise ValueError(f"Invalid multipart/form-data: {e}")
+
+        # Parse plain text
+        elif content_type == "text/plain":
+            return {"text": message.body()}
+
+        # Parse XML
+        elif content_type == "application/xml":
+            try:
+                _root = ElementTree.fromstring(message.body())
+                return {elem.tag: elem.text for elem in _root}
+            except ElementTree.ParseError as e:
+                raise ValueError(f"Invalid XML: {e}")
+
+        else:
+            raise ValueError(f"Unsupported Content-Type: {content_type}")

amqp/adapter/data_response_factory.py

@@ -0,0 +1,151 @@
+"""
+method to create, serialize and deserialize DataResponse message
+"""
+
+import base64
+import re
+from io import BytesIO
+
+from amqp.adapter.serializer import map_as_string, parse_map_string
+from amqp.model.model import AMQErrorMessage, DataMessageMagicByte, DataResponse
+from amqp.model.snowflake_id import SnowflakeId
+
+
+class DataResponseFactory:
+
+    @staticmethod
+    def create_async_response_message(_id: SnowflakeId) -> DataResponse:
+        """
+        Helper method to create standard OK response.
+        :param _id: ID
+        :return: Response message
+        """
+        return DataResponseFactory.of(
+            id=_id,
+            response_code=200,
+            content_type="application/text",
+            body="OK".encode("utf-8"),
+            trace_info={},
+            error="",
+            error_cause="",
+        )
+
+    @staticmethod
+    def of(
+        id: SnowflakeId,
+        response_code: int,
+        content_type: str,
+        body: bytes,
+        trace_info: dict[str, str],
+        error: str | None = None,
+        error_cause: str | None = None,
+    ) -> DataResponse:
+        """
+        Create the DataResponse message supplying all key values
+        :param id: SnowflakeID
+        :param response_code: HTTP response code
+        :param content_type: MIME content type
+        :param body: payload
+        :param trace_info: Jaeger trace info (id)
+        :return: DataResponseMessage object
+        """
+        return DataResponse(
+            magic=DataMessageMagicByte.HTTP_REQUEST.value,
+            id=id,
+            response_code=str(response_code),
+            content_type=content_type,
+            error=error,
+            error_cause=error_cause,
+            headers={},
+            trace_info=trace_info,
+            base64body=base64.b64encode(body),
+        )
+
+    @staticmethod
+    def of_error_message(
+        id: SnowflakeId,
+        error_message: AMQErrorMessage,
+        trace_info: dict[str, str],
+    ) -> DataResponse:
+        return DataResponseFactory.of(
+            id=id,
+            response_code=error_message.response_code,
+            content_type=error_message.content_type,
+            body=error_message.serialize(),
+            trace_info=trace_info,
+            error=error_message.error,
+            error_cause=error_message.detail,
+        )
+
+    @staticmethod
+    def serialize(response: DataResponse) -> bytes:
+        """
+        Serialize the message to byte array (to be sent over RabbitMQ)
+        :param response: Object to serialize
+        :return: byte array as serialized message
+        """
+        id_bytes = response.id().as_string().encode("utf-8")
+        prolog = (
+            f"|r={response.response_code()}|c={response.content_type()}|e={response.error()}|f={response.error_cause()}|t={{{map_as_string(response.trace_info())}}}|b="
+        ).encode("utf-8")
+        header_length = 1 + len(id_bytes) + len(prolog)
+        prolog_len = header_length.to_bytes(2, byteorder="big")
+        msg_length = 2 + header_length + len(response.base64body())
+        with BytesIO(bytearray(msg_length)) as baos:
+            baos.write(prolog_len)
+            baos.write(DataMessageMagicByte.HTTP_REQUEST.value.encode("utf-8"))
+            baos.write(id_bytes)
+            baos.write(prolog)
+            baos.write(response.base64body())
+            return baos.getvalue()
+
+    @staticmethod
+    def from_bytes(input_bytes: bytes) -> DataResponse:
+        """
+        Buil;ds the DataResponse object from its serialized form
+        :param input_bytes: byte array - serialized response
+        :return: DataResponse object
+        """
+        prolog_len = (input_bytes[0] << 8) + input_bytes[1]
+        metadata = input_bytes[2 : prolog_len + 2].decode()
+
+        pattern = re.compile(
+            f"{DataMessageMagicByte.HTTP_REQUEST.value}"
+            r"([0-9A-Fa-f]+)\|"
+            r"r=([^|]*)\|"
+            r"c=([^|]*)\|"
+            r"e=([^|]*)\|"
+            r"f=([^|]*)\|"
+            r"t=(\{[^}]*\})\|"
+            r"b="
+        )
+        match = pattern.match(metadata)
+        if not match:
+            raise ValueError(
+                f"DataResponse format does not match expected format. "
+                f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
+            )
+
+        id_str = match.group(1)
+        response_code = int(match.group(2))
+        content_type = match.group(3)
+        error = match.group(4)
+        error_cause = match.group(5)
+        trace_info_str = match.group(6)
+        body_b64 = input_bytes[prolog_len + 2 :]
+
+        trace_info = parse_map_string(trace_info_str)
+
+        return DataResponseFactory.of(
+            id_str,
+            response_code,
+            content_type,
+            base64.b64decode(body_b64),
+            trace_info,
+            error,
+            error_cause,
+        )
+
+
+# Ensure backward compatibility
+AMQResponseFactory = DataResponseFactory

amqp/adapter/file_handler.py

@@ -0,0 +1,415 @@
+"""
+implements the file downloader for the amqp adapter, where file is sent in chunks over RabbitMQ
+"""
+
+import asyncio
+import json
+import logging
+import os
+from asyncio import AbstractEventLoop
+from collections import defaultdict
+from pathlib import Path
+
+from aio_pika import Message, connect_robust
+from aio_pika.abc import (
+    AbstractExchange,
+    AbstractIncomingMessage,
+    AbstractQueue,
+    AbstractRobustChannel,
+    DeliveryMode,
+)
+
+from amqp.adapter.logging_utils import logging_debug, logging_error, logging_info
+from amqp.router.utils import await_result
+
+# Global dictionary to track completed downloads
+download_locations = defaultdict(str)
+IN_PROGRESS = 0
+END_OF_FILE = 1
+CANCELLED = 2
+
+
+class FileHandler:
+    """
+    The abstract base class for file handling. This is mainly to make the code testable, as there are only 2
+    options for pushing (uploading) a file(s) to a service: as HTTP multipart message delivered via RabbitMQ
+    or via stream in RabbitMQ. Downloading a file is always done via RabbitMQ stream. Thus the method signiture
+    is tighly coupled to the RabbitMQ API in v1. Only alternative implementation is a mock object in unittests.
+    """
+
+    def __init__(self):
+        pass
+
+    def ensure_directory_exists(self, filepath) -> str:
+        """
+        Ensures that the directory for the given filepath exists, and handles file versioning.
+
+        Args:
+            filepath: The path to the file.
+
+        Returns:
+            The original filepath if the file does not exist, or a modified filepath
+            pointing to the next available version of the file if it does.
+        """
+        # 1. Ensure directory exists
+        directory = os.path.dirname(filepath)
+        if directory and not os.path.exists(directory):
+            os.makedirs(directory)
+
+        # 2. Handle file versioning
+        if not os.path.exists(filepath):
+            return filepath  # File doesn't exist, return original path
+
+        # File exists, find next available version
+        base, ext = os.path.splitext(filepath)
+        version = 1
+        while True:
+            new_filepath = f"{base}.{version}{ext}"
+            if not os.path.exists(new_filepath):
+                return new_filepath  # Found a free version, return this path
+            version += 1
+
+    async def download_buffer(self, loop, rabbitmq_url, queue_name) -> (bytearray, int):
+        """
+        Asynchronously downloads a bytearray from RabbitMQ based on the provided file information.
+        It assumes entire array is in the first (and only) chunk
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            queue_name (str): The directory where the file should be saved.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def download_file(
+        self, loop, rabbitmq_url, file_info, message_data, output_dir
+    ) -> (int, int):
+        """
+        Asynchronously downloads a file from RabbitMQ based on the provided file information.
+        takes the file_info and message_data and output_dir, creates queue consumer and waits for the file
+        to be downloaded. File may consist of multiple chunks, so the function will wait until all chunks are received.
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            file_info (dict): A dictionary containing file metadata
+            message_data (dict): The entire message
+            output_dir (str): The directory where the file should be saved.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def on_message(
+        self,
+        message: AbstractIncomingMessage,
+        json_data: bytes,
+        rabbitmq_url: str,
+        loop,
+        output_dir="downloaded_files",
+    ) -> defaultdict:
+        """
+        Asynchronous callback function for handling messages from the 'amq-cleverswarm' queue.
+
+        Args:
+            message: The incoming RabbitMQ message.
+            rabbitmq_url: The RabbitMQ connection URL.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def publish_file(
+        self,
+        exchange: AbstractExchange,
+        file_path: Path,
+        routing_key: str,
+        loop: AbstractEventLoop,
+        max_chunk_size: int = 2**20,  # 1MB default chunk size
+        content_type: str = "application/octet-stream",  # Default content type
+        delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
+    ) -> tuple[str, int]:
+        """
+        Reads a file from the file system and publishes its content to a RabbitMQ exchange
+        in chunks.  It declares a queue with a random name, binds it to the specified
+        exchange, and uses the queue name as the routing key.
+
+        Args:
+            exchange: The aio_pika RobustChannel to use for communication with RabbitMQ.
+            file_path: The path to the file to publish.
+            max_chunk_size: The maximum size of each chunk (in bytes).
+            routing_key: The name of the RabbitMQ exchange routing key to publish to correct queue.
+            loop: correct event loop
+            max_chunk_size: chunk size to send
+            content_type: The content type of the file data.
+            delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT).
+
+        Returns:
+            The name of the randomly generated queue where the file content was published.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+
+# The StreamingFileHandler class is a subclass of FileHandler that implements the download_file method
+class StreamingFileHandler(FileHandler):
+    """
+    Handles file downloads from RabbitMQ streams.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    # This method MUST NOT be static.
+    # For testing purpose, we will mock this method and inject a mock connection.
+    async def _get_connection(self, rabbitmq_url, loop):
+        return await connect_robust(rabbitmq_url, loop=loop)
+
+    async def _download_from_queue_to_file(
+        self, rabbitmq_url, queue_name, filepath, loop
+    ) -> tuple[int, int] | None:
+        """
+        Internal method for downloading a file from a queue to disk.
+        Return (file size, eof status).
+        """
+        # Here the normal mock doesn't work since this method
+        # is async and will be run in another loop,
+        # out of the scope of method mock.
+        connection = await self._get_connection(rabbitmq_url, loop=loop)
+        _file_size = 0
+        _eof = IN_PROGRESS
+        try:
+            channel: AbstractRobustChannel = await connection.channel()
+            queue: AbstractQueue = await channel.get_queue(queue_name, ensure=False)
+            logging_debug(f"Awaiting all file chunks being downloaded for {filepath}")
+            async with queue.iterator() as queue_iter:
+                async for message in queue_iter:
+                    async with message.process():
+                        with open(filepath, "ab") as f:
+                            f.write(message.body)
+                        _file_size = _file_size + message.body_size
+                        _eof = int(message.headers.get("eof", END_OF_FILE))
+                        if _eof != IN_PROGRESS:
+                            break
+        except Exception as e:
+            logging_error(f"File download failed: {e}")
+        finally:
+            await connection.close()
+
+        return _file_size, _eof
+
+    async def download_file(
+        self, loop, rabbitmq_url, file_info, message_data, output_dir
+    ) -> (int, int):
+        """
+        Asynchronously downloads a file from RabbitMQ based on the provided file information.
+        takes the file_info and message_data and output_dir, creates queue consumer and waits for the file to be downloaded
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            file_info (dict): A dictionary containing file metadata
+            message_data (dict): The entire message
+            output_dir (str): The directory where the file should be saved.
+        """
+        filename = file_info["filename"]
+        size = file_info["size"]
+        stream_name = file_info["streamName"]
+        queue_name = file_info["queue_name"]
+        filepath = os.path.join(output_dir, filename)
+        filepath = self.ensure_directory_exists(
+            filepath
+        )  # if filename exists, create a new (next) version
+        download_locations[file_info["key"]] = filepath
+        filename = os.path.basename(
+            filepath
+        )  # if a new version was created, ensure to use that new version
+        file_info["filename"] = filename
+
+        logging_debug(
+            f"Downloading file: {filename} (size: {size} bytes) from stream: {stream_name}:{queue_name} to {filepath}"
+        )
+
+        if queue_name:
+            _local_file_size, _local_eof = await await_result(
+                asyncio.run_coroutine_threadsafe(
+                    self._download_from_queue_to_file(rabbitmq_url, queue_name, filepath, loop),
+                    loop=loop,
+                )
+            )
+            return _local_file_size, _local_eof
+
+        return 0, CANCELLED
+
+    async def _download_from_queue_to_buffer(
+        self, rabbitmq_url, queue_name, loop
+    ) -> tuple[bytearray, int]:
+        """
+        Internal method for downloading a file from a queue to RAM.
+        Return (bytearray, eof status).
+        """
+        _connection = await self._get_connection(rabbitmq_url, loop=loop)
+        _eof = IN_PROGRESS
+        _res = bytearray()
+        try:
+            channel: AbstractRobustChannel = await _connection.channel()
+            queue: AbstractQueue = await channel.get_queue(queue_name, ensure=False)
+            logging_debug(f"Awaiting all buffer chunks being downloaded for {queue_name}")
+            async with queue.iterator() as queue_iter:
+                async for message in queue_iter:
+                    async with message.process():
+                        _res.extend(message.body)
+                        _eof = message.headers.get("eof", END_OF_FILE)
+                        if _eof != IN_PROGRESS:
+                            break
+            logging_info(f"Finished downloading buffer {queue_name}")
+        except Exception as e:
+            logging_error(f"Downloading buffer: {e}")
+
+        await _connection.close()
+        return _res, _eof
+
+    async def download_buffer(self, loop, rabbitmq_url, queue_name: str | None) -> (bytearray, int):
+        """
+        Asynchronously downloads a bytearay from RabbitMQ based on the provided file information.
+        It assumes entire array is in the first (and only) chunk
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            queue_name (str): The directory where the file should be saved.
+        """
+        logging_debug(f"Downloading buffer from stream: {queue_name}")
+        if queue_name:
+            _local_res, _local_eof = await await_result(
+                asyncio.run_coroutine_threadsafe(
+                    self._download_from_queue_to_buffer(rabbitmq_url, queue_name, loop), loop=loop
+                )
+            )
+            return _local_res, _local_eof
+        return 0, CANCELLED
+
+    async def on_message(
+        self,
+        message: AbstractIncomingMessage,
+        json_data: bytes,
+        rabbitmq_url: str,
+        loop,
+        output_dir="downloaded_files",
+    ) -> defaultdict:
+        """
+        Asynchronous callback function for handling messages from the 'amq-cleverswarm' queue.
+
+        Args:
+            message: The incoming RabbitMQ message.
+            json_data: The JSON data from the message.
+            rabbitmq_url: The RabbitMQ connection URL.
+            loop: The asyncio event loop of the RabbitMQ API
+            output_dir
+        """
+        try:
+            _message_data = json.loads(message.body.decode())
+            _amq_message_data = json.loads(json_data.decode())
+
+            files_to_download = []
+            for _key, _file_type in _amq_message_data["files"].items():
+                if len(_file_type) > 0:
+                    files_info = _file_type[0]
+                    stream_name = files_info["streamName"]
+                    files_info["queue_name"] = (
+                        _message_data.get(stream_name, "") if stream_name else ""
+                    )
+                    files_info["key"] = _key
+                    files_to_download.append(files_info)
+
+            if not files_to_download:
+                logging_info(" [%s] No files to download in this message.", message.delivery_tag)
+                # await message.ack()
+                return defaultdict()
+
+            # Create a task for each file download
+            _tasks = []
+            for file_info in files_to_download:
+                logging_info(f" [{message.delivery_tag}] about to create task for {file_info}")
+                _task = asyncio.create_task(
+                    self.download_file(loop, rabbitmq_url, file_info, _message_data, output_dir)
+                )
+                _tasks.append(_task)
+            logging_info(
+                f" [{message.delivery_tag}] Waiting for all files to download...gather(*tasks)"
+            )
+            await asyncio.gather(*_tasks)  # Wait for all downloads to complete
+
+            logging_info(f"All files downloaded. d-tag: {message.delivery_tag}")
+        except Exception as e:
+            logging_error(f"Building DataMessage: {e}")
+            asyncio.run_coroutine_threadsafe(
+                message.ack(), loop=loop
+            )  # ACK -> ignore the message (do not redeliver)
+
+        return download_locations
+
+    async def publish_file(
+        self,
+        exchange: AbstractExchange,
+        file_path: Path,
+        routing_key: str,
+        loop: AbstractEventLoop,
+        max_chunk_size: int = 2**20,  # 1MB default chunk size
+        content_type: str = "application/octet-stream",  # Default content type
+        delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
+    ) -> tuple[str, int]:
+        """
+        Reads a file from the file system and publishes its content to a RabbitMQ exchange
+        in chunks.  It declares a queue with a random name, binds it to the specified
+        exchange, and uses the queue name as the routing key.
+
+        Args:
+            exchange: The aio_pika RobustChannel to use for communication with RabbitMQ.
+            file_path: The path to the file to publish.
+            max_chunk_size: The maximum size of each chunk (in bytes).
+            routing_key: The name of the RabbitMQ exchange routing key to publish to correct queue.
+            loop: correct event loop
+            max_chunk_size: chunk size to send
+            content_type: The content type of the file data.
+            delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT).
+
+        Returns:
+            The name of the randomly generated queue where the file content was published.
+        """
+        if not file_path.is_file():
+            raise FileNotFoundError(f"File not found: {file_path}")
+        _file_size = os.path.getsize(file_path)
+
+        with open(file_path, "rb") as f:
+            _offset = 0
+            while True:
+                _chunk = f.read(max_chunk_size)
+                if not _chunk:
+                    break  # End of file
+
+                message = Message(
+                    body=_chunk,
+                    content_type=content_type,
+                    delivery_mode=delivery_mode,
+                    headers={
+                        "file_name": file_path.name,
+                        "pos": _offset,
+                        "total_size": _file_size,
+                        "chunk_size": len(_chunk),
+                        "eof": (
+                            END_OF_FILE if max_chunk_size + _offset >= _file_size else IN_PROGRESS
+                        ),
+                    },
+                )
+                logging.debug(
+                    f"Going to Publish chunk {_offset}/{_file_size} to {exchange.name}:{routing_key}"
+                )
+                _future = asyncio.run_coroutine_threadsafe(
+                    exchange.publish(message, routing_key=routing_key), loop
+                )
+                while not _future.done():
+                    # sleep allows coroutine to execute. must wait for completion to ensure all chunks written in right order
+                    await asyncio.sleep(0.02)
+                    logging_debug(f"Waiting for chunk {_offset}/{_file_size} future= {_future}")
+
+                _offset += len(_chunk)
+                logging_debug(
+                    f"Published chunk {_offset}/{_file_size} to {exchange.name}:{routing_key}"
+                )
+
+        logging_debug(
+            f"File {file_path} published to RabbitMQ exchange {exchange.name}, queue {routing_key}"
+        )
+        return routing_key, _file_size

amqp/adapter/http_adapter.py

@@ -1,68 +0,0 @@
-import logging
-from typing import Dict, List
-from aiohttp import ClientSession, ClientResponse
-
-from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import AMQMessage
-
-
-class HttpAdapter:
-    def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession):
-        self.amq_configuration = amq_configuration
-        self.session = session
-        self.service_host = self.amq_configuration.dispatch.service_host
-        self.service_port = self.amq_configuration.dispatch.service_port
-
-    async def handle(self, message: AMQMessage) -> ClientResponse:
-        method = message.method.lower()
-        if method == 'get':
-            return await self.get(message)
-        elif method == 'put':
-            return await self.put(message)
-        elif method == 'post':
-            return await self.post(message)
-        elif method == 'head':
-            return await self.head(message)
-        elif method == 'delete':
-            return await self.delete(message)
-        else:
-            raise ValueError(f"Unexpected HTTP method: {message.method}")
-
-    async def get(self, message: AMQMessage) -> ClientResponse:
-        url = f"http://{self.service_host}:{self.service_port}{message.path}"
-        headers = {**message.headers, **message.trace_info}
-        logging.info(f" [*] SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}")
-        return await self.session.get(url, headers=headers)
-
-    async def put(self, message: AMQMessage) -> ClientResponse:
-        return await self.handle_possible_form(
-            self.session.put(
-                f"http://{self.service_host}:{self.service_port}{message.path}",
-                data=message.body,
-                headers={**message.headers, **message.trace_info},
-            )
-        )
-
-    async def post(self, message: AMQMessage) -> ClientResponse:
-        return await self.handle_possible_form(
-            self.session.post(
-                f"http://{self.service_host}:{self.service_port}{message.path}",
-                data=message.body,
-                headers={**message.headers, **message.trace_info},
-            )
-        )
-
-    async def handle_possible_form(self, request_coroutine) -> ClientResponse:
-        return await request_coroutine
-
-    async def head(self, message: AMQMessage) -> ClientResponse:
-        raise NotImplementedError
-
-    async def delete(self, message: AMQMessage) -> ClientResponse:
-        raise NotImplementedError
-
-    def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
-        for header, values in message_headers.items():
-            if header.lower() == 'content-type':
-                return values[0].split(';')[0]
-        return 'application/json'

amqp/adapter/logging_utils.py

@@ -0,0 +1,128 @@
+import inspect
+import logging
+import os
+import sys
+import threading
+import traceback
+from importlib.metadata import version
+
+# Get version from installed package metadata
+__version__ = version("amq_adapter")  # Name must match pyproject.toml [project] name
+__verbose__ = os.environ.get("AMQ_VERBOSE_LOGGING", "0") == "1"
+
+
+def get_context_info():
+    """
+    Retrieves the current module, line number, and function name.
+
+    Returns:
+        A tuple containing (module name, line number, function name).
+        Returns (None, None, None) if it fails to retrieve the information.
+    """
+    try:
+        # Get the frame object for the current function call
+        frame = inspect.currentframe()
+        if frame is not None:
+            # Get the frame object for the caller of the current function
+            frame = frame.f_back.f_back
+        if frame:
+            code_context = frame.f_code
+            module_name = code_context.co_filename
+            module_name = (
+                "amqp" + module_name.split("/amqp", 1)[-1]
+                if "/amqp" in module_name
+                else module_name
+            )
+            line_number = frame.f_lineno
+            function_name = code_context.co_name
+            return module_name, line_number, function_name
+        else:
+            return None, None, None
+    except Exception:
+        return None, None, None
+
+
+def logging_error(msg: str, *args) -> None:
+    """
+    Log an error message according to current logging for 'ERROR' level.
+    Add module name, line and function name where the error occurred, on single line.
+    Args:
+        msg (str): The error message to log.
+    """
+    _exec_info = sys.exc_info()[2]
+    _thread_id = threading.get_ident()
+    if _exec_info:
+        _tb = traceback.extract_tb(sys.exc_info()[2])[-1]
+        logging.error(
+            f"{_thread_id}:{__version__} [E] '{msg}' ---> '{_tb.filename}':{_tb.lineno}, '{_tb.name}()'",
+            *args,
+        )
+    else:
+        module, line_number, function_name = get_context_info()
+        if module and line_number and function_name:
+            logging.error(
+                f"{_thread_id}:{__version__} [E] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+                *args,
+            )
+        else:
+            logging.error(f"{_thread_id}:{__version__} [E] {msg}", *args)
+
+
+def logging_warning(msg: str, *args) -> None:
+    """
+    Log a warning message according to current logging for 'WARN' level.
+    Add module name, line and function name where the print occurred, on single line.
+    Args:
+        msg (str): The warning message to log.
+    """
+    _thread_id = threading.get_ident()
+    if not __verbose__:
+        logging.warning(f"{_thread_id}:{__version__} [W] {msg}", *args)
+        return
+    module, line_number, function_name = get_context_info()
+    if module and line_number and function_name:
+        logging.warning(
+            f"{_thread_id}:{__version__} [W] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+            *args,
+        )
+    else:
+        logging.warning(f"{_thread_id}:{__version__} [W] {msg}", *args)
+
+
+def logging_info(msg: str, *args) -> None:
+    """
+    Log an info message according to current logging for 'INFO' level.
+    Add module name, line and function name where the print occurred, on single line.
+    Args:
+        msg (str): The info message to log.
+    """
+    _thread_id = threading.get_ident()
+    if not __verbose__:
+        logging.info(f"{_thread_id}:{__version__} [*] {msg}", *args)
+        return
+    module, line_number, function_name = get_context_info()
+    if module and line_number and function_name:
+        logging.info(
+            f"{_thread_id}:{__version__} [*] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+            *args,
+        )
+    else:
+        logging.info(f"{_thread_id}:{__version__} [*] {msg}", *args)
+
+
+def logging_debug(msg: str, *args) -> None:
+    """
+    Log a debug message according to current logging for 'DEBUG' level.
+    Add module name, line and function name where the print occurred, on single line.
+    Args:
+        msg (str): The debug message to log.
+    """
+    _thread_id = threading.get_ident()
+    module, line_number, function_name = get_context_info()
+    if module and line_number and function_name:
+        logging.debug(
+            f"{_thread_id}:{__version__} [DBG] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+            *args,
+        )
+    else:
+        logging.debug(f"{_thread_id}:{__version__} [DBG] {msg}", *args)

amqp/adapter/pydantic_serializer.py

@@ -0,0 +1,182 @@
+"""
+This module provides serialization and deserialization functions for Pydantic models that extend BaseModel
+"""
+
+import importlib
+import json
+import uuid
+from datetime import date, datetime
+from types import UnionType
+from typing import Any, Dict, Union, get_args, get_origin
+from urllib.parse import parse_qs, urlparse
+
+from pydantic import BaseModel
+
+from amqp.adapter.logging_utils import logging_debug
+
+
+def serialize_object(obj: BaseModel) -> str:
+    """
+    Serializes an R2RSerializable object to a string in the format
+    "<ClassName>:{key1:value1, key2:value2,...}".  Handles UUIDs and datetimes,
+    and nested Pydantic models.
+
+    Args:
+        obj: The BaseModel object to serialize.
+
+    Returns:
+        A string representation of the object.
+    """
+    class_name = obj.__class__.__name__
+    module_name = obj.__class__.__module__
+    obj_dict = {}
+    for key, _ in obj.model_dump().items():
+        # here we use the original value
+        # the one returned by model_dump will convert everything into dict recursively
+        value = getattr(obj, key)
+        if isinstance(value, uuid.UUID):
+            obj_dict[key] = str(value)
+        elif isinstance(value, datetime):
+            obj_dict[key] = value.isoformat()
+        elif isinstance(value, dict):  # Ensure dictionaries are handled
+            obj_dict[key] = value
+        elif isinstance(value, BaseModel):
+            obj_dict[key] = serialize_object(value)  # Recursively serialize
+        elif isinstance(value, list):
+            serialized_list = []
+            for item in value:
+                if isinstance(item, BaseModel):
+                    serialized_list.append(serialize_object(item))
+                else:
+                    serialized_list.append(item)
+            obj_dict[key] = serialized_list
+        else:
+            obj_dict[key] = value
+
+    dict_string = json.dumps(obj_dict)
+    return f"{module_name}.{class_name}:{dict_string}"
+
+
+def python_type_to_json(data):
+    """
+    Convert Python objects (dict, list, tuple, set, datetime, etc.) to a JSON string.
+    Handles common non-JSON-serializable types like datetime, date, and sets.
+
+    Args:
+        data: Python object to be converted (dict, list, tuple, set, etc.)
+
+    Returns:
+        str: JSON string representation of the input data
+
+    Raises:
+        TypeError: If the object contains non-serializable types (e.g., custom classes)
+    """
+
+    def default_serializer(obj):
+        """Custom serializer for non-JSON-serializable objects."""
+        if isinstance(obj, (datetime, date)):
+            return obj.isoformat()  # Convert datetime/date to ISO string
+        elif isinstance(obj, set):
+            return list(obj)  # Convert sets to lists
+        elif hasattr(obj, "__dict__"):
+            return obj.__dict__  # Attempt to serialize custom objects
+        else:
+            raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+    return json.dumps(data, default=default_serializer, indent=0, ensure_ascii=False)
+
+
+def try_deserialize_object(data: str) -> BaseModel | str:
+    """
+    Try deserialize object, if failed, return the input string as-is.
+    """
+    if not isinstance(data, str):
+        return data
+    try:
+        return deserialize_object(data)
+    except Exception as e:
+        logging_debug(f"Failed to deserialize object: {e}")
+        return data
+
+
+def deserialize_object(data: str) -> BaseModel:
+    """
+    Deserializes a string in the format "<ClassName>:{key1:value1, key2:value2,...}"
+    back into an BaseModel object.  Handles UUIDs, datetimes, and
+    nested Pydantic models.
+
+    Args:
+        data: The string to deserialize.
+
+    Returns:
+        An R2RSerializable object.
+    """
+    class_path, dict_str = data.split(":", 1)
+    module_path, class_name = class_path.rsplit(".", 1)
+
+    obj_dict: Dict[str, Any] = json.loads(dict_str)
+
+    logging_debug(f"Deserialized object: {class_path}")
+    if "[" in class_name:
+        class_name = class_name.split("[")[0]
+    logging_debug(f"trying object: {class_name}")
+    module = importlib.import_module(module_path)
+    cls = getattr(module, class_name)
+    if cls is None:
+        raise ValueError(f"Unknown class: {class_name}")
+
+    # process nested model, dict and list
+    for key, value in obj_dict.items():
+        key_type: type[Any] = cls.model_fields[key].annotation
+        # handle union with none, this will give the true type
+        # for example, return `A` if the type is `A | None`
+        if get_origin(key_type) is UnionType or get_origin(key_type) is Union:
+            union_types = get_args(key_type)
+            if len(union_types) > 2:
+                # if there are multiple unions, like `A | B | None`
+                # then we have no way to figure out which one to use
+                raise ValueError(f"Unsupported Union type: {key_type}")
+            # return the none-None one as the type for processing
+            if union_types[0] is type(None):
+                key_type = union_types[1]
+            else:
+                key_type = union_types[0]
+        # skip if the value is None
+        if value is None:
+            continue
+
+        if key_type == uuid.UUID:
+            obj_dict[key] = uuid.UUID(value)
+        elif key_type == datetime:
+            obj_dict[key] = datetime.fromisoformat(value)
+        elif key_type == dict:
+            obj_dict[key] = value
+        elif issubclass(key_type, BaseModel):
+            obj_dict[key] = deserialize_object(value)
+        elif key_type == list:
+            obj_dict[key] = list(map(try_deserialize_object, value))
+
+    return cls(**obj_dict)
+
+
+def parse_url_query(url):
+    """
+    Parses a HTTP GET URL and returns (context_path, query_params_dict)
+
+    Args:
+        url: The URL string to parse (e.g., "http://example.com/path?foo=1&bar=2&foo=3")
+
+    Returns:
+        tuple: (context_path, query_params_dict)
+        - context_path: The path part of the URL (e.g., "/path")
+        - query_params_dict: Dictionary of query parameters where values are always lists
+                           (e.g., {"foo": ["1", "3"], "bar": ["2"]})
+    """
+    parsed = urlparse(url)
+    query_params = parse_qs(parsed.query)
+
+    # Convert values from lists to single values when there's only one value
+    # If you prefer to always keep values as lists, remove this comprehension
+    simplified_params = {k: v[0] if len(v) == 1 else v for k, v in query_params.items()}
+
+    return (parsed.path, simplified_params)

amqp/adapter/response_holder.py

@@ -1,15 +0,0 @@
-from dataclasses import dataclass
-from concurrent.futures import Future
-
-from aiohttp.web_response import Response
-from opentelemetry.trace import Span
-
-
-@dataclass
-class ResponseHolder:
-    response: Future[Response]
-    tracing_span: Span
-
-    def __init__(self, response: Future[Response], span: Span):
-        self.response = response
-        self.tracing_span = span

amqp/adapter/serializer.py

@@ -0,0 +1,72 @@
+"""
+Helper functions used to aid with (de)serialization of messages sent over AMQP.
+"""
+
+import struct
+from io import BytesIO
+from typing import Dict, List, Union
+
+
+def long_to_bytes(x: int) -> bytes:
+    return struct.pack("q", x)
+
+
+def bytes_to_long(b: bytes) -> int:
+    return struct.unpack("q", b)[0]
+
+
+def read_long(bis: BytesIO) -> int:
+    try:
+        return struct.unpack("q", bis.read(8))[0]
+    except struct.error:
+        return 0
+
+
+def object_or_list_as_string(value: Union[str, List[str]]) -> str:
+    if isinstance(value, list):
+        return f"[{','.join(value)}]"
+    return str(value)
+
+
+def map_as_string(map_: Dict[str, str]) -> str:
+    return ",".join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
+
+
+def map_with_list_as_string(map_: Dict[str, List[str]]) -> str:
+    return ",".join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
+
+
+def parse_map_string(input_str: str) -> Dict[str, str]:
+    map_ = {}
+    if input_str.startswith("{") and input_str.endswith("}"):
+        for entry in input_str[1:-1].split(","):
+            if len(entry) > 1:
+                key, value = entry.split("=")
+                map_[key] = value
+    return map_
+
+
+def csv_as_list(input_str: str) -> List[str]:
+    return [item.strip() for item in input_str.strip("[]").split(",")]
+
+
+def add_to_map(map_: Dict[str, List[str]], token: str) -> None:
+    try:
+        eq_idx = token.index("=")
+    except ValueError:
+        eq_idx = -1  # set to -1 when not found
+    if eq_idx > 0:
+        key = token[:eq_idx]
+        map_[key] = csv_as_list(token[eq_idx + 1 :])
+
+
+def parse_map_list_string(input_str: str) -> Dict[str, List[str]]:
+    map_ = {}
+    if input_str.startswith("{") and input_str.endswith("}"):
+        for token in input_str[1:-1].split("],"):
+            add_to_map(map_, token)
+    return map_
+
+
+def str_to_bool(value: str) -> bool:
+    return value.lower() in ("true", "yes", "1", "t", "y", "on")

amqp/adapter/service_message_factory.py

@@ -1,59 +1,91 @@
 import base64
-import logging
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.logging_utils import logging_error, logging_info
+from amqp.adapter.serializer import map_as_string, parse_map_string
 from amqp.adapter.trace_info_adapter import TraceInfoAdapter
-from amqp.model.model import CleverMicroServiceMessage, ServiceMessageType
+from amqp.model.model import ServiceMessage
+from amqp.model.service_message_type import ServiceMessageType
 from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import parse_map_string, map_as_string
 
 RS = "|"
 SPLIT_RS = r"\|"
 
 
-INVALID_MESSAGE = CleverMicroServiceMessage()  # default values mean invalid message
+INVALID_MESSAGE = ServiceMessage()  # default values mean invalid message
 
 
-class CleverMicroServiceMessageFactory:
+class ServiceMessageFactory:
+    """
+    build/serialize/deserialize CleverMicro Service Message class
+    """
 
     def __init__(self, name: str):
         self.process_name = name
 
-    def to_string(self, message: CleverMicroServiceMessage) -> str:
+    def to_string(self, message: ServiceMessage) -> str:
+        """
+        Convert ServiceMessage to human-readable format
+        :param message: ServiceMessage object
+        :return: as string
+        """
         if message is not None:
             return (
-                f"CleverMicroServiceMessage{{id={message.id.as_string()}|type={self.format_message_type(message)}|"
+                f"ServiceMessage{{id={message.id.as_string()}|type={self.format_message_type(message)}|"
                 f"recipient={message.recipient_name}|replyTo={message.reply_to}|"
                 f"traceInfo={message.trace_info}|body={base64.b64encode(message.message.encode()).decode() if message.payload_type & 0x80 else message.message}}}"
             )
         return "null"
 
-    def of(self, message_type: ServiceMessageType, payload: str, recipient_name: str) -> CleverMicroServiceMessage:
-        _id = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        return CleverMicroServiceMessage(
+    def of(
+        self, message_type: ServiceMessageType, payload: str, recipient_name: str
+    ) -> ServiceMessage:
+        """
+        Builds ServiceMessage object from relevant values.
+        :param message_type: enum Message type
+        :param payload: Message content, stored as-is
+        :param recipient_name: Recipient name (from the routing expression)
+        :return: ServiceMessage  object
+        """
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        return ServiceMessage(
             id=_id,
             payload_type=0,
             message_type=message_type,
             recipient_name=recipient_name,
             reply_to=self.process_name,
             trace_info=TraceInfoAdapter.create_produce_span(_id),
-            message=payload
+            message=payload,
         )
 
-    def as_base64(self, message_type: ServiceMessageType, payload: str, recipient_name: str) -> CleverMicroServiceMessage:
-        _id = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        return CleverMicroServiceMessage(
+    def as_base64(
+        self, message_type: ServiceMessageType, payload: str, recipient_name: str
+    ) -> ServiceMessage:
+        """
+        Builds ServiceMessage object from relevant values, stores content Base64 encoded.
+        :param message_type: enum Message type
+        :param payload: Message content, stored as Base64 string
+        :param recipient_name: Recipient name (from the routing expression)
+        :return: ServiceMessage  object
+        """
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        return ServiceMessage(
             id=_id,
             payload_type=0x80,
             message_type=message_type,
             recipient_name=recipient_name,
             reply_to=self.process_name,
             trace_info=TraceInfoAdapter.create_produce_span(_id.as_string()),
-            message=payload
+            message=payload,
         )
 
-    def from_bytes(self, input_bytes: bytes) -> CleverMicroServiceMessage:
-        print(f"CleverMicroServiceMessage >>> [{input_bytes.decode()}]")
+    def from_bytes(self, input_bytes: bytes) -> ServiceMessage:
+        """
+        Deserialize ServiceMessage from byte array serialized form.
+        :param input_bytes: serialized ServiceMessage
+        :return: ServiceMessage object represented by the input.
+        """
+        logging_info(f"ServiceMessage >>> [{input_bytes.decode()}]")
         input_str_array = input_bytes.decode().split(RS)
         try:
             i = 0
@@ -69,39 +101,47 @@ class CleverMicroServiceMessageFactory:
             trace_info = parse_map_string(input_str_array[i]) if i < len(input_str_array) else {}
             i += 1
             message = input_str_array[i] if i < len(input_str_array) else ""
-            return CleverMicroServiceMessage(
+            return ServiceMessage(
                 id=SnowflakeId.from_hex(_id),
                 payload_type=payload_type,
                 message_type=message_type,
                 recipient_name=recipient_name,
                 reply_to=reply_to,
                 trace_info=trace_info,
-                message=base64.b64decode(message).decode() if payload_type & 0x80 else message
+                message=(base64.b64decode(message).decode() if payload_type & 0x80 else message),
             )
         except Exception as e:
-            logging.error(
+            logging_error(
                 "Unexpected ServiceMessage format. Expected 6 tokens separated by '|' in format: "
                 "[ID(64bit number)|MessageType(8bit number)|RecipientName(String)|"
                 "ReplyTo(String)|TraceInfo|Payload(String or Base64)], got: "
-                "[{}], reported problem: {}", input_bytes.decode(), str(e)
+                "[%s], reported problem: %s",
+                input_bytes.decode(),
+                str(e),
             )
             return INVALID_MESSAGE
 
-    def serialize(self, message: CleverMicroServiceMessage) -> bytes:
+    def serialize(self, message: ServiceMessage) -> bytes:
+        """
+        Serialize ServiceMessage to byte array.
+        :param message: ServiceMessage to serialize.
+        :return: serialized object.
+        """
         return (
             f"{message.id.as_string()}{RS}{self.format_message_type(message)}{RS}{message.recipient_name}{RS}"
             f"{message.reply_to}{RS}{map_as_string(message.trace_info)}{RS}"
             f"{base64.b64encode(message.message.encode()).decode() if message.payload_type != 0 else message.message}"
-        ).encode('utf-8')
+        ).encode("utf-8")
 
-
-    def format_message_type(self, message: CleverMicroServiceMessage) -> str:
+    def format_message_type(self, message: ServiceMessage) -> str:
         """
-        Formats the Enum message type and ecorates it with base64 flag at highest bit.
+        Formats the Enum message type and decorates it with base64 flag at highest bit.
         For compatibility with 0-based enum ordinals in Java subtract 1 from ordinal value.
         :param message: Service message to send
         :return: formatted message type
         """
-        num_type = (message.message_type.value if message.message_type else ServiceMessageType.UNKNOWN.value) - 1
-        fmt = ((num_type & 0x7F) | (message.payload_type & 0x80))
+        num_type = (
+            message.message_type.value if message.message_type else ServiceMessageType.UNKNOWN.value
+        ) - 1
+        fmt = (num_type & 0x7F) | (message.payload_type & 0x80)
         return f"{fmt:02X}"

amqp/adapter/test_amq_message_factory.py

@@ -1,68 +0,0 @@
-from datetime import datetime, timezone
-from unittest import TestCase
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-
-
-class AMQMessageFactoryTest(TestCase):
-    def test_from_bytes(self):
-        raw = "\002\027A64f3b5af4d00000a4000|m=POST|d=clever-pybook.localhost|p=/debug|h={Accept=[*/*],X-Forwarded-Host=[clever-pybook.localhost:8085],User-Agent=[curl/8.7.1],X-Forwarded-Proto=[http],X-Forwarded-For=[172.18.0.1],Host=[clever-pybook.localhost:8085],Accept-Encoding=[gzip],Content-Length=[1475],X-Forwarded-Port=[8085],X-Real-Ip=[172.18.0.1],X-Forwarded-Server=[c5945bef67d1],Content-Type=[multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v]}|t={traceparent=00-bed6eb7de0d8ccf4c80b145a4bfec357-8d1f3bafc471bf0f-01}|b=TEST_ME"
-        msg = AMQMessageFactory.from_bytes(raw.encode("utf-8"))
-        self.assertIsNotNone(msg.id)
-        self.assertEqual(msg.id.as_string(),'64F3B5AF4D00000A4000','SnowflakeID incorrectly parsed.')
-        self.assertEqual(msg.headers["Content-Type"][0], "multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v")
-
-    def test_generate_snowflake_id(self):
-        _id = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        self.assertEqual('00001000', _id.as_string()[-8:], 'Instance ID not set correctly')
-
-        _id = AMQMessageFactory(2).generate_snowflake_id()
-        self.assertEqual('00002000', _id.as_string()[-8:], 'Instance ID not set correctly')
-
-        _f = AMQMessageFactory(127)
-        _i0 = _f.generate_snowflake_id()
-        _i1 = _f.generate_snowflake_id()
-        _i2 = _f.generate_snowflake_id()
-        _i3 = _f.generate_snowflake_id()
-        self.assertEqual('0007F000', _i0.as_string()[-8:], 'Instance ID and sequence not set correctly')
-        self.assertEqual('0007F001', _i1.as_string()[-8:], 'Instance ID and sequence not set correctly')
-        self.assertEqual('0007F002', _i2.as_string()[-8:], 'Instance ID and sequence not set correctly')
-        self.assertEqual('0007F003', _i3.as_string()[-8:], 'Instance ID and sequence not set correctly')
-
-    def test_create_request_message(self):
-        _f = AMQMessageFactory(34)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        self.assertEqual("localhost", _req.domain, 'Domain not set correctly')
-        self.assertEqual("/test/me", _req.path, 'Context path not set correctly')
-        self.assertEqual("v1", _req.headers['k1'][0], "Headers not set correctly")
-        self.assertEqual(b'Quick brown fox', _req.body(), "Body not correctly decoded")
-
-    def test_serialize(self):
-        _f = AMQMessageFactory(34)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        _ser = _f.serialize(_req)
-        self.assertEqual(65, _ser[2])
-        _deser = _f.from_bytes(_ser)
-        self.assertEqual(_req.id.as_string(), _deser.id.as_string())
-        self.assertEqual("localhost", _deser.domain)
-        self.assertEqual("/test/me", _deser.path)
-        self.assertEqual(b"Quick brown fox", _deser.body())
-        self.assertEqual("v2", _deser.headers["k2"][0])
-
-    def test_amq_message_routing_key(self):
-        _f = AMQMessageFactory(35)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        _key = _f.amq_message_routing_key(_req)
-        self.assertEqual("localhost..test.me", _key, "Message routing key wrongly extracted from the message")
-
-    def test_get_timestamp_from_id(self):
-        _f = AMQMessageFactory(35)
-        _timestamp = _f.get_timestamp_from_id(_f.generate_snowflake_id())
-        _x_timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
-        self.assertLessEqual(_x_timestamp - _timestamp, 1, "Timestamp in SnowflakeId differs too much from current time")
-
-    def test_to_string(self):
-        _f = AMQMessageFactory(35)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        _as_str = _f.to_string(_req)
-        self.assertGreater(_as_str.index("domain='localhost'"), 0, "Expected content not found")

amqp/adapter/test_amq_response_factory.py

@@ -1,36 +0,0 @@
-from unittest import TestCase
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-from amqp.adapter.amq_response_factory import AMQResponseFactory
-from amqp.model.snowflake_id import SnowflakeId
-
-
-class TestAMQResponseFactory(TestCase):
-
-    def test_create_async_response_message(self):
-        _f = AMQResponseFactory()
-        _g = AMQMessageFactory(111)
-        _resp = _f.create_async_response_message(_g.generate_snowflake_id())
-        self.assertEqual(200, _resp.response_code)
-        self.assertEqual(b"OK", _resp.response)
-
-    def test_serialize(self):
-        _f = AMQResponseFactory()
-        _g = AMQMessageFactory(111)
-        _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
-        _resp = _f.create_async_response_message(_snowflakeId)
-        _ser = _f.serialize(_resp)
-        self.assertEqual(2, str(_ser[2:]).index(_f.MAGIC_BYTE_HTTP_REQUEST+_snowflakeId.as_string()), "Magic byte and SnowflakeId not found")
-        self.assertGreater(str(_ser[2:]).index("|e=|"), 2)
-        self.assertEqual((_ser[0]+_ser[1]), len(_ser) - 2 - len('T0s='))
-
-    def test_from_bytes(self):
-        _f = AMQResponseFactory()
-        _g = AMQMessageFactory(111)
-        _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
-        _resp = _f.create_async_response_message(_snowflakeId)
-        _ser = _f.serialize(_resp)
-        _deser = _f.from_bytes(_ser)
-        self.assertEqual(_snowflakeId.as_string(), _deser.id)
-        self.assertEqual(200, _deser.response_code)
-        self.assertEqual(b'OK', _deser.response)

amqp/adapter/test_amq_route_factory.py

@@ -1,20 +0,0 @@
-from unittest import TestCase
-
-from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
-from amqp.model.model import AMQRoute
-
-
-class TestAMQRouteFactory(TestCase):
-    def test_from_string(self):
-        _f = AMQRouteFactory()
-        _r: AMQRoute = _f.from_string("amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0")
-        self.assertEqual("", _r.exchange)
-        self.assertEqual("amq-clever-pybook", _r.queue)
-        self.assertEqual(5, _r.timeout)
-        _r2: AMQRoute = _f.from_string("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#:5")
-        self.assertEqual(NULL_ROUTE, _r2)
-
-    def test_validate(self):
-        _f = AMQRouteFactory()
-        self.assertTrue(_f.validate("amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"))
-        self.assertFalse(_f.validate("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#"))

amqp/adapter/trace_info_adapter.py

@@ -1,14 +1,13 @@
 from dataclasses import dataclass
 from typing import Dict
 
+
 @dataclass
 class TraceInfoAdapter:
     """
-    This class should set up the Open Telemetry tracing stack. To be done later.
+    TODO This class should set up the Open Telemetry tracing stack. To be done later.
     """
 
     @staticmethod
     def create_produce_span(message_id: str) -> Dict[str, str]:
-        return {
-            "trace-id": message_id
-        }
+        return {"trace-id": message_id}

amqp/adapter/type_descriptor.py

@@ -0,0 +1,205 @@
+""" """
+
+import importlib
+import json
+from typing import Any
+
+
+class TypeDescriptor:
+    def __init__(self, type_string: str, extra_init_data: dict):
+        self.type_string = type_string
+        self.extra_init_data = extra_init_data or {}
+        self.is_json: bool = False
+        self.type: str = self._extract_type()
+
+    def _extract_type(self) -> str:
+        if self.type_string.startswith("<class '"):
+            return self.type_string[8:-2]
+        # Handle Annotated case of FastAPI annotation
+        if "typing.Optional" in self.type_string:
+            annotated = _extract_type_of(self.type_string)
+            if "typing.Annotated" in annotated:
+                # Extract content between the first square brackets after Annotated
+                match = _extract_type_of(annotated)
+                if match:
+                    # Split by comma and get first part, then strip whitespace
+                    parts = [p.strip() for p in match.split(",")]
+                    if len(parts) > 1:
+                        self.is_json = parts[1].lower() == "json"
+                    return parts[0] if parts else ""
+            else:
+                # Handle simple Optional case
+                return annotated
+        return _extract_type_of(self.type_string)
+
+    def instantiate(self, payload: Any, is_json: bool = False) -> Any:
+        return _instantiate(
+            (
+                self.type
+                if "typing.Optional" in self.type_string or self.type_string.startswith("<class ")
+                else self.type_string
+            ),
+            payload,
+            self.is_json or is_json,
+            extra_init_data=self.extra_init_data,
+        )
+
+    def __repr__(self) -> str:
+        return f"TypeDescriptor(type='{self.type}', is_json={self.is_json})"
+
+
+def _extract_type_of(input_string: str) -> str:
+    stack = []
+    start_index = -1
+    end_index = -1
+
+    for i, char in enumerate(input_string):
+        if i > 0:  # this assumes paramerized type, that must be at lest 1 char long
+            if char == "[":
+                if start_index == -1:
+                    start_index = i
+                stack.append(char)
+            elif char == "]":
+                if len(stack) == 1:
+                    end_index = i
+                    break
+                if len(stack) > 0:
+                    stack.pop()
+
+    if start_index != -1 and end_index != -1:
+        return input_string[start_index + 1 : end_index].strip()
+    else:
+        return input_string.strip() if start_index == 1 and end_index == -1 else ""
+
+
+def merge_extra_init_data(cls, json_payload, extra_init_data: dict):
+    """
+    Merges extra_init_data into json_payload only when:
+    1. cls has an attribute named like the key
+    2. json_payload doesn't already have the key
+    """
+    return {
+        **json_payload,
+        **{
+            k: v
+            for k, v in extra_init_data.items()
+            if (hasattr(cls, k) or (hasattr(cls, "model_fields") and k in cls.model_fields))
+            and k not in json_payload
+        },
+    }
+
+
+def _instantiate(
+    type_string: str, payload: Any, is_json: bool = False, extra_init_data: dict = None
+) -> Any:
+    """
+    Dynamically imports and instantiates a class from a TypeDescriptor's type_string attribute.
+
+    Args:
+        type_string: TypeDescriptor instance with the class path
+        payload: Argument to pass to the constructor
+
+    Returns:
+        Instance of the target class initialized with payload
+
+    Raises:
+        ImportError: If the module/class cannot be imported
+        TypeError: If instantiation fails
+    """
+    # Handle JSON payload first
+    json_payload = json.loads(payload) if is_json and isinstance(payload, str) else None
+
+    # Handle basic types
+    if type_string == "str":
+        return str(payload)
+    if type_string == "int":
+        return int(payload)
+    if type_string == "float":
+        return float(payload)
+    if type_string == "bool":
+        return bool(payload)
+
+    # Handle typing module types
+    if type_string.startswith("typing.") or type_string.startswith("list"):
+        if (
+            type_string.startswith("typing.List")
+            or type_string.startswith("typing.Sequence")
+            or type_string.startswith("list")
+        ):
+            if json_payload is not None:
+                return json_payload
+            inner_type = _extract_type_of(type_string)
+            if inner_type:
+                return [
+                    _instantiate(inner_type, p, extra_init_data=extra_init_data)
+                    for p in (payload if isinstance(payload, list) else payload.split(","))
+                ]
+            return payload if isinstance(payload, list) else payload.split(",")
+
+        if type_string.startswith("typing.Dict") or type_string.startswith("typing.Mapping"):
+            if json_payload is not None:
+                return json_payload
+            return dict(payload)
+
+        if type_string.startswith("typing.Set"):
+            if json_payload is not None:
+                return set(json_payload)
+            _set_type = _extract_type_of(type_string)
+            return set(
+                payload
+                if isinstance(payload, list)
+                else [_instantiate(_set_type, p) for p in payload.split(",")]
+            )
+
+        if type_string.startswith("typing.Tuple"):
+            if json_payload is not None:
+                return tuple(json_payload)
+            _tuple_type = _extract_type_of(type_string)
+            return tuple(
+                payload
+                if isinstance(payload, list)
+                else [_instantiate(_tuple_type, p) for p in payload.split(",")]
+            )
+
+        if type_string.startswith("typing.Optional"):
+            inner_type = _extract_type_of(type_string)
+            if inner_type:
+                return _instantiate(inner_type, payload, is_json, extra_init_data)
+            return payload
+
+    # Handle regular class instantiation
+    parts = type_string.split(".")
+    module_path = ".".join(parts[:-1])
+    class_name = parts[-1]
+
+    try:
+        # Import the module
+        module = importlib.import_module(module_path)
+        # Get the class
+        cls = getattr(module, class_name)
+        if isinstance(payload, cls):
+            return payload
+        # Instantiate with payload
+        if json_payload is not None:
+            _args = merge_extra_init_data(cls, json_payload, extra_init_data or {})
+            if hasattr(cls, "from_dict"):
+                return cls.from_dict(_args)
+            elif hasattr(cls, "create"):
+                return cls.create(**_args)
+            return cls(**_args)
+
+        if isinstance(payload, dict):
+            _args = merge_extra_init_data(cls, payload, extra_init_data or {})
+            if hasattr(cls, "from_dict"):
+                return cls.from_dict(_args)
+            elif hasattr(cls, "create"):
+                return cls.create(**_args)
+            return cls(**_args)
+
+        return cls(payload)
+    except ImportError as e:
+        raise ImportError(f"Could not import module '{module_path}'") from e
+    except AttributeError as e:
+        raise ImportError(f"Module '{module_path}' has no class '{class_name}'") from e
+    except TypeError as e:
+        raise TypeError(f"Failed to instantiate {type_string} with payload {payload}") from e

amqp/config/amq_configuration.py

@@ -1,10 +1,20 @@
 import configparser
+import inspect
 import os
 
+from amqp.adapter.logging_utils import logging_error, logging_warning
+
+"""
+Convert configuration from properties file to an object.
+"""
+
 
 class AMQAdapter:
+    """
+    configuration with prefix 'cm.amq-adapter'
+    """
 
-    PREFIX: str = 'cm.amq-adapter.'
+    PREFIX: str = "cm.amq-adapter."
 
     def __init__(self, config: configparser.ConfigParser):
         """
@@ -14,23 +24,40 @@ class AMQAdapter:
 
         :param config: config parser to read configuration values
         """
-        self.generator_id = config.getint('CleverMicro-AMQ', self.PREFIX + 'generator-id', fallback=164)
-        self.service_name = config.get('CleverMicro-AMQ', self.PREFIX + 'service-name', fallback='amq-python-adapter-'+self.generator_id)
-        self.route_mapping = config.get('CleverMicro-AMQ', self.PREFIX + 'route-mapping', fallback=None)
+        self.generator_id = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "generator-id", fallback=164
+        )
+        self.service_name = config.get(
+            "CleverMicro-AMQ",
+            self.PREFIX + "service-name",
+            fallback="amq-python-adapter-" + str(self.generator_id),
+        )
+        self.route_mapping = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "route-mapping", fallback=None
+        )
+        self.require_authenticated_user = config.getboolean(
+            "CleverMicro-AMQ",
+            self.PREFIX + "require-authenticated-user",
+            fallback=False,
+        )
+        # these 2 shall be provided by Docker swarm, format is a random alphanumeric string
+        self.swarm_service_id = os.getenv("SWARM_SERVICE_ID", default="clever-amqp-service")
+        self.swarm_task_id = os.getenv("SWARM_TASK_ID", default="python-adapter")
 
     def adapter_prefix(self) -> str:
         return f"{self.service_name}-{self.generator_id}-"
 
 
-
 class Dispatch:
+    """
+    configuration with prefix 'cm.dispatch'
+    """
 
-    PREFIX: str = 'cm.dispatch.'
+    PREFIX: str = "cm.dispatch."
 
     def __init__(self, config: configparser.ConfigParser):
         """
         cm.dispatch.use-dlq=true
-        cm.dispatch.use-confirms=false
         cm.dispatch.amq-host=rabbitmq
         cm.dispatch.amq-port=5672
         cm.dispatch.amq-port-tls=5671
@@ -39,40 +66,97 @@ class Dispatch:
 
         :param config: config parser to read configuration values
         """
-        self.use_dlq = config.getboolean('CleverMicro-AMQ', self.PREFIX + 'use-dlq', fallback=False)
-        self.use_confirms = config.getboolean('CleverMicro-AMQ', self.PREFIX + 'use-confirms', fallback=False)
-        self.amq_host = config.get('CleverMicro-AMQ', self.PREFIX + 'amq-host', fallback='rabbitmq')
-        self.amq_port = config.getint('CleverMicro-AMQ', self.PREFIX + 'amq-port', fallback=5672)
-        self.amq_port_tls = config.getint('CleverMicro-AMQ', self.PREFIX + 'amq-port-tls', fallback=5671)
-        self.service_host = config.get('CleverMicro-AMQ', self.PREFIX + 'service-host', fallback='rabbitmq')
-        self.service_port = config.getint('CleverMicro-AMQ', self.PREFIX + 'service-port', fallback=8080)
-        self.rabbit_mq_user = os.environ["RABBIT_MQ_USER"]
-        self.rabbit_mq_password = os.environ["RABBIT_MQ_PASSWORD"]
+        self.use_dlq = config.getboolean("CleverMicro-AMQ", self.PREFIX + "use-dlq", fallback=False)
+        self.amq_host = config.get("CleverMicro-AMQ", self.PREFIX + "amq-host", fallback="rabbitmq")
+        self.amq_port = config.getint("CleverMicro-AMQ", self.PREFIX + "amq-port", fallback=5672)
+        self.amq_port_tls = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "amq-port-tls", fallback=5671
+        )
+        self.service_host = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "service-host", fallback="localhost"
+        )
+        self.service_port = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "service-port", fallback=8080
+        )
+        self.rabbit_mq_user = os.getenv("RABBIT_MQ_USER", "springuser")
+        self.rabbit_mq_password = os.getenv("RABBIT_MQ_PASSWORD", "TheCleverWho")
+        self.rabbit_mq_url = (
+            f"amqp://{self.rabbit_mq_user}:{self.rabbit_mq_password}@{self.amq_host}/"
+        )
+        self.download_dir = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "download-dir", fallback="downloaded_files"
+        )
 
 
 class Backpressure:
+    """
+    configuration with prefix 'cm.backpressure'
+    """
 
-    PREFIX: str = 'cm.backpressure.'
+    PREFIX: str = "cm.backpressure."
 
     def __init__(self, config: configparser.ConfigParser):
         """
         cm.backpressure.threshold=20
         cm.backpressure.time-window=3000
-        cm.backpressure.management-service-url=management-service:8080
 
         :param config: config parser to read configuration values
         """
-        self.management_service_url = config.get('CleverMicro-AMQ', self.PREFIX + 'management-service-url', fallback='management-service:8080')
-        self.threshold = config.getint('CleverMicro-AMQ', self.PREFIX + 'threshold', fallback=20)
-        self.time_window = config.getint('CleverMicro-AMQ', self.PREFIX + 'time-window', fallback=3000)
+        self.threshold_threads = config.getint(
+            "CleverMicro-AMQ",
+            self.PREFIX + "threshold",
+            fallback=int(os.getenv("PARALLEL_WORKERS", 1)),
+        )
+        # time-window in seconds, duration between backpressure reports
+        self.time_window = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "time-window", fallback=10
+        )
+
+        # Define maximum CPU usage before triggering backpressure OVERLOAD alert
+        self.threshold_cpu = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "threshold-cpu", fallback=90
+        )
+        # Define the time window length, in seconds, during which, if CPU usage is consistently above the threshold, the system will trigger OVERLOAD alert
+        self.cpu_overload_duration = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "cpu-overload-duration", fallback=30
+        )
+        # Define the time window length, in seconds, during which, if no request is made to the Service, the system will trigger IDLE alert
+        self.idle_duration = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "idle-duration", fallback=30
+        )
 
 
 class AMQConfiguration:
+    """
+    Top level configuration context holder / object
+    """
 
-    def __init__(self):
+    def __init__(self, config_file_name: str = None):
         config = configparser.ConfigParser()
         # Read the application.properties file
-        config.read('config/application.properties')
+        if os.path.exists(config_file_name):
+            config.read(config_file_name)
+        else:
+            # Get the directory of the caller (the module where this function is called)
+            caller_frame = inspect.stack()[1]  # 0 is current frame, 1 is caller
+            caller_module = inspect.getmodule(caller_frame[0])
+            if caller_module:
+                caller_dir = os.path.dirname(os.path.abspath(caller_module.__file__))
+                config_file_path = os.path.join(caller_dir, config_file_name)
+                config.read(config_file_path)
+            else:
+                logging_error(f"Configuration file not found: {config_file_name}")
+                logging_warning(
+                    "Defaulting to preset values, which will likely be wrong, causing errors."
+                )
+
+        # Override values with environment variables
+        for section in config.sections():
+            for option in config.options(section):
+                # env_var = f"{section.upper()}_{option.upper()}"
+                if option in os.environ:
+                    config.set(section, option, os.environ[option])
+
         #
         # Add individual config areas
         #

amqp/config/application.properties

@@ -2,17 +2,41 @@
 # CleverMicro AMQ Adapter settings
 # ====================================================================
 [CleverMicro-AMQ]
-cm.amq-adapter.service-name=amq-adapter
-cm.amq-adapter.is-router=false
-cm.amq-adapter.generator-id=164
-cm.amq-adapter.route-mapping=
+# CleverMicro AMQ Adapter settings that identify this AMQ Adapter instance to other adapters
 
+# A name to identify this service for the AMQ Adapter
+cm.amq-adapter.service-name=cleverthis-service-name
+# The CleverMicro AMQ Adapter instance ID, used to identify this instance in the CleverMicro AMQ Adapter cluster
+cm.amq-adapter.generator-id=1
+# Route mapping. For route detail description see section 3.1 in Onboarding Guide for Python:
+# https://docs.cleverthis.com/en/architecture/microservices/onboarding/python
+# NOTE: this is a comma-separated list of route mappings (multiple, comma-separated routes are allowed here)
+cm.amq-adapter.route-mapping=
+# Indicate if AMQ Adapter should respond with HTTP 401 Unauthorized if the request does not contain valid JWT token
+cm.amq-adapter.require-authenticated-user=false
+
+# CleverMicro AMQ Adapter dispatcher, settings related to RabbitMQ connection and optional REST forwarding
 cm.dispatch.use-dlq=true
-cm.dispatch.use-confirms=false
 cm.dispatch.amq-host=rabbitmq
 cm.dispatch.amq-port=5672
 cm.dispatch.amq-port-tls=5671
+cm.dispatch.download-dir=/tmp/downloaded_files
+# Not used for CleveSwarm nor CleverBRAG or in tight coupling mode.
+# applicable only in loose coupling mode, specify the destination where to forward the REST request
+#cm.dispatch.service-host=cleverthis-service-name.localhost
+#cm.dispatch.service-port=8080
+
+# Backpressure monitor settings
+# Define maximum number of messages of being processed in parallel before triggering backpressure OVERLOAD alert
+cm.backpressure.threshold=5
+# Define maximum CPU usage (%) before triggering backpressure OVERLOAD alert
+cm.backpressure.threshold-cpu-overload=90
+# Define maximum CPU usage (%) before triggering backpressure IDLE alert
+cm.backpressure.threshold-cpu-idle=10
+# Define the time window between the Backpressure reports, in seconds
+cm.backpressure.time-window=10
+# Define the time window length, in seconds, during which, if CPU usage is consistently above the threshold, the system will trigger OVERLOAD alert
+cm.backpressure.cpu-overload-duration=30
+# Define the time window length, in seconds, during which, if no request is made to the Service, the system will trigger IDLE alert
+cm.backpressure.cpu-idle-duration=30
 
-cm.backpressure.threshold=20
-cm.backpressure.time-window=3000
-cm.backpressure.management-service-url=management-service:8080

amqp/config/application.properties.local

@@ -0,0 +1,19 @@
+# ====================================================================
+# CleverMicro AMQ Adapter settings
+# ====================================================================
+[CleverMicro-AMQ]
+cm.amq-adapter.service-name=amq-adapter-python
+cm.amq-adapter.generator-id=211
+cm.amq-adapter.route-mapping=amq-cleverswarm::amq-cleverswarm:localhost.#:5:0
+
+cm.dispatch.use-dlq=true
+cm.dispatch.use-confirms=false
+cm.dispatch.amq-host=localhost
+cm.dispatch.amq-port=5672
+cm.dispatch.amq-port-tls=5671
+# Not used for CleverSwarm nor CleverBRAG or in tight coupling mode.
+# applicable only in loose coupling mode, specify the destination where to forward the REST request
+#cm.dispatch.service-host=cleverswarm.localhost
+#cm.dispatch.service-port=8080
+
+cm.backpressure.threshold=20

amqp/model/model.py

@@ -1,100 +1,281 @@
 import base64
-
-from typing import List, Mapping, Dict
+import json
 from dataclasses import dataclass, field
-from enum import Enum, auto
+from enum import Enum
+from typing import Dict, List
 
+from aio_pika.abc import AbstractRobustConnection
+
+from amqp.model.service_message_type import ServiceMessageType
 from amqp.model.snowflake_id import SnowflakeId
 from amqp.router.utils import sanitize_as_rabbitmq_name
 
 RS = ":"
 
 
+"""
+Define the structure of messages used in Clever Micro
+"""
+
+
+class DataMessageMagicByte(Enum):
+    HTTP_REQUEST = "A"
+    RPC_REQUEST = "R"
+
+
+@dataclass
+class CleverMicroMessage:
+    DEFAULT_CONTENT_TYPE = ["application/octet-stream"]
+
+    """
+    The common structure of a message that transfers the user request to the Service and back. All data related messages
+    conform to this base structure:
+
+    magic - identifies the message type (REST, RPC, DataResponse)
+    id - unique message ID
+    m_field - method name (for REST style calls) or service name (for RPC style calls) or response code for DataResponse
+    d_field - domain name (for REST style calls) or module name (for RPC style calls) or  error code for DataResponse
+    p_field - path (for REST style calls) or method name (for RPC style calls) or error cause for DataResponse
+    headers - headers of the message (e.g. Content-Type, Accept, etc.)
+    trace_info - trace information (e.g. trace ID, span ID, etc. for Jaeger trace monitor)
+    base64body - the body of the message encoded in base64
+    """
+
+    def __init__(
+        self,
+        magic: str,
+        id: SnowflakeId,
+        m_field: str,
+        d_field: str,
+        p_field: str,
+        headers: Dict[str, List[str]],
+        trace_info: Dict[str, str],
+        base64body: bytes,
+    ):
+        self._magic = magic
+        self._id: SnowflakeId = id
+        self._m_field = m_field
+        self._d_field = d_field
+        self._p_field = p_field
+        self._headers: Dict[str, List[str]] = headers
+        self._trace_info: Dict[str, str] = trace_info
+        self._base64body: bytes = base64body
+
+    def magic(self):
+        return self._magic
+
+    def id(self):
+        return self._id
+
+    def m_field(self) -> str:
+        return self._m_field
+
+    def d_field(self) -> str:
+        return self._d_field
+
+    def p_field(self) -> str:
+        return self._p_field
+
+    def headers(self) -> Dict[str, List[str]]:
+        return self._headers
+
+    def trace_info(self) -> Dict[str, str]:
+        return self._trace_info
+
+    def base64body(self) -> bytes:
+        return self._base64body
+
+    def body(self) -> bytes:
+        return base64.b64decode(self.base64body())
+
+    def content_type(self) -> str:
+        return self.headers().get("Content-Type", self.DEFAULT_CONTENT_TYPE)[0]
+
+
+class DataMessage(CleverMicroMessage):
+    """
+    The class represents a structure of a message that transfers the user request to the Service.
+    It may also be used to make a call/request between services.
+    The response to this call is in format of DataResponse class.
+
+    See DataMessageFactory for code to instantiate, serialize and deserialize the class
+    """
+
+    def __init__(
+        self,
+        magic: str,
+        id: SnowflakeId,
+        method: str,
+        domain: str,
+        path: str,
+        headers: Dict[str, List[str]],
+        trace_info: Dict[str, str],
+        base64body: bytes,
+    ):
+        super().__init__(magic, id, method, domain, path, headers, trace_info, base64body)
+
+    def method(self) -> str:
+        return self.m_field()
+
+    def domain(self) -> str:
+        return self.d_field()
+
+    def path(self) -> str:
+        return self.p_field()
+
+    def routing_key(self) -> str:
+        return sanitize_as_rabbitmq_name(f"{self.d_field()}.{self.p_field()}")
+
+
+class MultipartDataMessage(DataMessage):
+    def __init__(self, message: DataMessage, extra_data: dict):
+        super().__init__(
+            message.magic(),
+            message.id(),
+            message.method(),
+            message.domain(),
+            message.path(),
+            message.headers(),
+            message.trace_info(),
+            message.base64body(),
+        )
+        self.extra_data = extra_data
+
+
+@dataclass
+class AMQMessage(DataMessage):
+    """
+    Expand the core DataMessage type to include the channel, which is required to create a dedicated queue
+    in case the response should initiate file download.
+    """
+
+    _connection: AbstractRobustConnection = field(repr=False)  # Don't show in repr
+
+    def __init__(self, data_message: DataMessage, connection: AbstractRobustConnection):
+        # Initialize the parent DataMessage with all properties from the input data_message
+        super().__init__(
+            magic=data_message.magic(),
+            id=data_message.id(),
+            method=data_message.method(),
+            domain=data_message.domain(),
+            path=data_message.path(),
+            headers=data_message.headers(),
+            trace_info=data_message.trace_info(),
+            base64body=data_message.base64body(),
+        )
+        self._connection = connection
+        self.extra_data = data_message.extra_data if hasattr(data_message, "extra_data") else {}
+
+    @property
+    def connection(self) -> AbstractRobustConnection:
+        """Get the AMQP connection associated with this message"""
+        return self._connection
+
+
+class DataResponse(CleverMicroMessage):
+    """
+    The response message to REST & RPC style calls (note: the request is made with DataMessage)
+
+    See DataResponseFactory for code to instantiate, serialize and deserialize the class
+    """
+
+    def __init__(
+        self,
+        magic: str,
+        id: SnowflakeId,
+        response_code: str,
+        content_type: str,
+        error: str,
+        error_cause: str,
+        headers: Dict[str, List[str]],
+        trace_info: Dict[str, str],
+        base64body: bytes,
+    ):
+        if headers is None:
+            headers = {}
+        headers["Content-Type"] = [content_type]
+        super().__init__(
+            magic,
+            id,
+            response_code,
+            error,
+            error_cause,
+            headers,
+            trace_info,
+            base64body,
+        )
+
+    def response_code(self) -> str:
+        return self.m_field()
+
+    def error(self) -> str:
+        return self.d_field()
+
+    def error_cause(self) -> str:
+        return self.p_field()
+
+
 @dataclass(frozen=True)
 class AMQRoute:
     """
     Configuration item representing one AMQRoute. Adapter uses this to declare the Exchange
     and/or queue and bind the key as routing expression.
 
-    :param componentName: a service/component name that this route points to (owner of the route)
+    :param component_name: a service/component name that this route points to (owner of the route)
     :param exchange: RabbitMQ exchange name
     :param queue: RabbitMQ queue name
     :param key: RabbitMQ TOPIC Exchange routing key
     :param timeout: Timeout in milliseconds, how long to wait for reply, or not waiting if less then 1
-    :param validUntil: optional timestamp (millis from epoch) after which the route is ignored
+    :param valid_until: optional timestamp (millis from epoch) after which the route is ignored
     """
-    componentName: str
+
+    component_name: str
     exchange: str
     queue: str
     key: str
     timeout: int
-    validUntil: int
+    valid_until: int
 
     FORMAT = "Component-Name:Exchange-Name:Queue-Name:Routing-Key:Timeout:Valid-Until"
 
     def __str__(self):
         return (
-            f"{self.componentName}{RS}{self.exchange}{RS}{self.queue}{RS}{self.key}"
-            f"{RS}{self.timeout}{RS}{self.validUntil}"
+            f"{self.component_name}{RS}{self.exchange}{RS}{self.queue}{RS}{self.key}"
+            f"{RS}{self.timeout}{RS}{self.valid_until}"
         )
 
     def __eq__(self, other):
         if isinstance(other, AMQRoute):
-            return (self.componentName == other.componentName
-                    and self.queue == other.queue
-                    and self.key == other.key)
+            return (
+                self.component_name == other.component_name
+                and self.queue == other.queue
+                and self.key == other.key
+            )
         return False
 
     @property
     def primary_key(self):
-        return f"{self.componentName}{RS}{self.queue}{RS}{self.key}"
+        """
+        Build the unique identifier of the route
+        :return primary key
+        """
+        return f"{self.component_name}{RS}{self.queue}{RS}{self.key}"
 
     def as_string(self) -> str:
         return self.__str__()
 
 
-@dataclass
-class AMQMessage:
-    magic: str
-    id: SnowflakeId
-    method: str
-    domain: str
-    path: str
-    headers: Mapping[str, List[str]]
-    trace_info: Mapping[str, str]
-    base64_body: bytes
-
-    def routing_key(self) -> str:
-        return sanitize_as_rabbitmq_name(f"{self.domain}.{self.path}")
-
-    def body(self) -> bytes:
-        return base64.b64decode(self.base64_body)
-
-
-@dataclass
-class AMQResponse:
-    id: str
-    response_code: int
-    content_type: str
-    response: bytes
-    error: str
-    error_cause: str
-    trace_info: Mapping[str, str]
-
-    def body(self) -> bytes:
-        return base64.b64decode(self.response)
-
-
-class ServiceMessageType(Enum):
-    ROUTING_DATA_REQUEST = auto()
-    ROUTING_DATA = auto()
-    ROUTING_DATA_REMOVE = auto()
-    ROUTING_DATA_REPLACE = auto()
-    UNKNOWN = auto()
+# Ensure backward compatibility
+AMQResponse = DataResponse
 
 
 @dataclass(frozen=True)
-class CleverMicroServiceMessage:
+class ServiceMessage:
+    """
+    The CleverMicro service message that is used to communicate the internal state of the AMQ channel.
+    In v0.2 it supports service discovery / route registration and Backpressure status notification.
+    """
+
     id: SnowflakeId = None
     payload_type: int = 0
     message_type: ServiceMessageType = ServiceMessageType.UNKNOWN
@@ -105,3 +286,74 @@ class CleverMicroServiceMessage:
 
     def is_valid(self) -> bool:
         return self.message_type is not None
+
+
+class ScalingRequestAlert(Enum):
+    OVERLOAD = "OVERLOAD"
+    IDLE = "IDLE"
+    UPDATE = "UPDATE"
+    SCALE_UP = "SCALE_UP"
+    SCALE_DOWN = "SCALE_DOWN"
+
+
+@dataclass(frozen=True)
+class ScalingRequest:
+    service_id: str
+    task_id: str
+    max_availability: int
+    current_availability: int
+    request_type: ScalingRequestAlert
+    version: int = 1
+
+    def to_json(self) -> str:
+        """Converts the object to a JSON string matching the Java structure"""
+        return json.dumps(
+            {
+                "version": self.version,
+                "serviceId": self.service_id,
+                "taskId": self.task_id,
+                "maxAvailability": self.max_availability,
+                "currentAvailability": self.current_availability,
+                "requestType": self.request_type.value,  # Using .value for Enum serialization
+            },
+            indent=2,
+        )
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "ScalingRequest":
+        """Creates a ScalingRequest from a JSON string"""
+        data = json.loads(json_str)
+        return cls(
+            service_id=data["serviceId"],
+            task_id=data["taskId"],
+            max_availability=data["maxAvailability"],
+            current_availability=data["currentAvailability"],
+            request_type=ScalingRequestAlert(data["requestType"]),  # Convert string to Enum
+        )
+
+
+class AMQErrorMessage:
+    """
+    The error response message to RPC style call (note: the request is made with DataMessage)
+    """
+
+    def __init__(
+        self,
+        response_code: int,
+        error: str,
+        detail: str,
+        content_type: str = "application/json",
+    ):
+        self.response_code = response_code
+        self.error = error
+        self.detail = detail
+        self.content_type = content_type
+
+    def serialize(self) -> bytes:
+        return json.dumps(
+            {
+                "response_code": self.response_code,
+                "error": self.error,
+                "detail": self.detail,
+            }
+        ).encode("utf-8")

amqp/model/service_message_type.py

@@ -0,0 +1,10 @@
+from enum import Enum, auto
+
+
+class ServiceMessageType(Enum):
+    ROUTING_DATA_REQUEST = auto()
+    ROUTING_DATA = auto()
+    ROUTING_DATA_REMOVE = auto()
+    ROUTING_DATA_REPLACE = auto()
+    BACKPRESSURE = auto()
+    UNKNOWN = auto()

amqp/model/snowflake_id.py

@@ -2,25 +2,31 @@ import struct
 
 
 class SnowflakeId:
-    SNOWFLAKE_ID_WIDTH = 10  # with in bytes (8 bits each), can be more than long int width (8 bytes)
+    """
+    see https://en.wikipedia.org/wiki/Snowflake_ID
+    """
 
-    def __init__(self, _input: bytes = None):
+    SNOWFLAKE_ID_WIDTH = (
+        10  # with in bytes (8 bits each), can be more than long int width (8 bytes)
+    )
+
+    def __init__(self, hi_bits_or_bytes=None, lo_bits=None):
         self.bits = [0, 0]
-        if _input is not None:
-            i = 0
-            shift = 8  # starting from Hi value, which is only 2 bytes wide
-            n = 1
-            while i < len(_input) and i <= self.SNOWFLAKE_ID_WIDTH:
-                self.bits[n] |= _input[i] << shift
-                if shift == 0:
-                    n -= 1
-                    shift = 56
-                else:
-                    shift -= 8
-                i += 1
-
-    def __init__(self, hi_bits: int, lo_bits: int):
-        self.bits = [lo_bits, hi_bits]
+        if lo_bits is None:
+            if hi_bits_or_bytes is not None:
+                i = 0
+                shift = 8  # starting from Hi value, which is only 2 bytes wide
+                n = 1
+                while i < len(hi_bits_or_bytes) and i <= self.SNOWFLAKE_ID_WIDTH:
+                    self.bits[n] |= hi_bits_or_bytes[i] << shift
+                    if shift == 0:
+                        n -= 1
+                        shift = 56
+                    else:
+                        shift -= 8
+                    i += 1
+        else:
+            self.bits = [lo_bits, hi_bits_or_bytes]
 
     @staticmethod
     def from_hex(_input):
@@ -36,13 +42,18 @@ class SnowflakeId:
         for i in range(0, expected_length, 2):
             if i == expected_length - (2 * 8):
                 n = 0
-            bits[n] = (bits[n] << 8) | int(_input[i:i + 2], 16)
+            bits[n] = (bits[n] << 8) | int(_input[i : i + 2], 16)
 
         return SnowflakeId(bits[1], bits[0])
 
     def __str__(self):
         _hex = f"{self.bits[1]:016X}{self.bits[0]:016X}"
-        return _hex if len(_hex) <= 2 * SnowflakeId.SNOWFLAKE_ID_WIDTH else _hex[(-2 * SnowflakeId.SNOWFLAKE_ID_WIDTH):]
+        _val = (
+            _hex
+            if len(_hex) <= 2 * SnowflakeId.SNOWFLAKE_ID_WIDTH
+            else _hex[(-2 * SnowflakeId.SNOWFLAKE_ID_WIDTH) :]
+        )
+        return _val.lower()
 
     def __eq__(self, other):
         if isinstance(other, SnowflakeId):
@@ -59,7 +70,7 @@ class SnowflakeId:
     def get_bytes(self):
         high = struct.pack("!Q", self.bits[1])
         low = struct.pack("!Q", self.bits[0])
-        return high[2:] + low
+        return high[-2:] + low
 
     def get_bits(self):
         return self.bits

amqp/rabbitmq/rabbit_mq_client.py

@@ -0,0 +1,228 @@
+import logging
+
+import aio_pika
+from aio_pika.abc import AbstractRobustExchange, AbstractRobustQueue
+from pika.exchange_type import ExchangeType
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.logging_utils import (
+    logging_debug,
+    logging_error,
+    logging_info,
+    logging_warning,
+)
+from amqp.model.model import AMQRoute, DataMessage
+from amqp.router.router_base import (
+    AMQ_REPLY_TO_EXCHANGE,
+    AMQ_RPC_EXCHANGE,
+    DLQ_EXCHANGE,
+)
+from amqp.router.router_consumer import RouterConsumer
+from amqp.router.utils import sanitize_as_rabbitmq_name
+
+
+class RabbitMQClient:
+    PREFETCH_COUNT = 1
+    DLQ_ARGS = {"x-dead-letter-exchange": DLQ_EXCHANGE}
+
+    def __init__(self, configuration):
+        self.channel: aio_pika.RobustChannel | None = None
+        self.connection: aio_pika.RobustConnection | None = None
+        self.rpc_exchange: aio_pika.RobustExchange | None = None
+        self.reply_exchange: aio_pika.RobustExchange | None = None
+        self.amq_configuration = configuration
+        logging.info("*******************************************")
+        logging.info(
+            "******** RabbitMQClient.init(): %s",
+            self.amq_configuration.dispatch.amq_host,
+        )
+        logging.info("*******************************************")
+        self.router = RouterConsumer(configuration)
+        # this.meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
+
+        self.reply_to_exchange: AbstractRobustExchange | None = (
+            None  # when sending response to RPC request
+        )
+        self.reply_to_queue: AbstractRobustQueue | None = (
+            None  # when receiving response to RPC request
+        )
+
+    async def init(self, loop) -> AbstractRobustExchange:
+        await self.setup_amq_channel(loop)
+        await self.router.init_routing_paths_consumer(self.router.channel)
+        cm_reply_to_queue = self.router.get_reply_to_queue_name()
+        self.reply_to_queue = await self.channel.declare_queue(
+            name=cm_reply_to_queue, durable=True, exclusive=False, auto_delete=False
+        )
+        self.reply_to_exchange = await self.channel.declare_exchange(
+            AMQ_REPLY_TO_EXCHANGE, type=ExchangeType.direct
+        )
+        await self.reply_to_queue.bind(
+            exchange=self.reply_to_exchange, routing_key=cm_reply_to_queue
+        )
+        return self.reply_to_exchange
+
+    def cleanup(self):
+        logging_info("RabbitMQClient.cleanup()")
+        self.router.cleanup()  # de-register route(s)
+        if self.router.is_open(self.channel):
+            try:
+                self.channel.close()
+                if not self.connection.is_closed:
+                    self.connection.close()
+            except Exception as e:
+                logging_error(
+                    "RabbitMQClient.PreDestroy cleanup - connection close error: %s",
+                    e,
+                )
+
+    async def register_inbound_routes(self, route_mapping: str, callback):
+        """
+        Register inbound routes for the AMQ adapter. The route mapping is a comma-separated string that contains
+        list of routes. A route is the mapping of domain and path to the exchange and queue name.
+        This method declare (and thus create) the queue and exchange, bind them together using the
+        regex-like routing expression given in the route.
+        After this the Service is listening on (consuming from) the queue for incoming messages.
+        """
+        if self.router.is_open(self.channel):
+            queue_args = self.DLQ_ARGS if self.amq_configuration.dispatch.use_dlq else None
+            logging_info("RabbitMQClient register routes, cfg mapping: [%s]", route_mapping)
+            # convert the comma-separated string into list of AMQRoute objects
+            requested_routes = self.router.unwrap_route_list(route_mapping)
+            for route in requested_routes:
+                try:
+                    # Note: AMQRoute allows one of the exchange or queue to be None, in which case the default is used
+                    queue_name = (
+                        route.queue if route.queue else self.router.get_consume_queue_name()
+                    )
+                    logging_info("RabbitMQClient register route: [%s]", route)
+                    queue: AbstractRobustQueue = await self.channel.declare_queue(
+                        name=queue_name,
+                        durable=True,
+                        exclusive=False,
+                        auto_delete=False,
+                        arguments=queue_args,
+                    )
+                    _exchange_name: str = route.exchange if route.exchange else AMQ_RPC_EXCHANGE
+                    exchange: AbstractRobustExchange = await self.channel.declare_exchange(
+                        name=_exchange_name, type=ExchangeType.topic
+                    )
+                    await queue.bind(exchange, route.key)
+                    _c_tag = await queue.consume(callback, no_ack=False)
+                    # Register the route with the router and publish its detail to other adapters
+                    await self.router.register_route(
+                        AMQRoute(
+                            route.component_name,
+                            _exchange_name,
+                            queue_name,
+                            sanitize_as_rabbitmq_name(route.key),
+                            route.timeout,
+                            route.valid_until,
+                        )
+                    )
+                    logging_info(
+                        " [%s] AMQ connection initialized for route: %s, listens on: %s/%s",
+                        _c_tag,
+                        route,
+                        exchange,
+                        queue_name,
+                    )
+                except Exception as err:
+                    logging_error("Callback registration INCOMPLETE, %s", err)
+        else:
+            logging_warning("Channel is null, cannot register routes.")
+
+    async def register_data_reply_callback(self, rpc_callback):
+        _c_tag = await self.reply_to_queue.consume(callback=rpc_callback, no_ack=False)
+        logging_info(
+            " [%s] RabbitMQClient register data reply callback on queue: %s",
+            _c_tag,
+            self.reply_to_queue.name,
+        )
+
+    async def setup_amq_channel(self, loop):
+        """
+        Set up the RabbitMQ connection
+        :return: nothing, it applies changes to instance variables
+        """
+        self.connection = await aio_pika.connect_robust(
+            host=self.amq_configuration.dispatch.amq_host,
+            port=self.amq_configuration.dispatch.amq_port,
+            login=self.amq_configuration.dispatch.rabbit_mq_user,
+            password=self.amq_configuration.dispatch.rabbit_mq_password,
+            loop=loop,
+        )
+        logging_debug(
+            "RabbitMQClient.setupAMQ, got connection, open=%s",
+            self.connection.is_closed,
+        )
+        self.channel = await self.connection.channel()
+        await self.channel.set_qos(prefetch_count=self.PREFETCH_COUNT)
+        await self.router.init_internal_routing_paths(self.channel)
+        self.router.set_route_added_notifier(
+            lambda route: logging_debug(
+                " RouteAdded, setting up metrics: amqp.adapter.exchange.%s",
+                route.exchange,
+            )
+        )
+
+    async def send_message(self, message: DataMessage):
+        """
+        The API routine that sends a datagram message (DataMessage) to the destination. The Destination
+        is determined from the DataMessage metadata section, using `domain` and `path` as key.
+        :param message: message to send
+        :return: nothing (void) as submitting message to RabbitMQ exchange does not return anything
+         meaningful. There will be asynch ACK later that will confirm the message was sent to RMQ.
+        """
+        try:
+            route = self.router.find_route_by_message(message)
+            if route and self.rpc_exchange:
+                if route.timeout <= 0:
+                    # No response expected
+                    pika_message: aio_pika.message.Message = aio_pika.message.Message(
+                        body=DataMessageFactory.serialize(message),
+                        content_encoding="utf-8",
+                        delivery_mode=2,
+                    )
+                    await self.rpc_exchange.publish(
+                        message=pika_message,
+                        routing_key=self.router.get_routing_key(
+                            message
+                        ),  # context path is a routing key
+                    )
+                    logging_info(f"Msg Publish (no-reply), messageId: {message.id()}, to: {route}")
+                else:
+                    # This is RPC call, there should be response
+                    pika_message: aio_pika.message.Message = aio_pika.message.Message(
+                        body=DataMessageFactory.serialize(message),
+                        content_encoding="utf-8",
+                        delivery_mode=2,
+                        correlation_id=str(message.id()),
+                        reply_to=self.router.get_reply_to_queue_name(),
+                    )
+                    await self.rpc_exchange.publish(
+                        message=pika_message,
+                        routing_key=self.router.get_routing_key(
+                            message
+                        ),  # context path is a routing key
+                    )
+                    logging_info(
+                        "Msg Publish (RPC call), messageId: %s, to: %s",
+                        message.id().as_string(),
+                        route.as_string(),
+                    )
+            else:
+                logging_warning(
+                    "Message not sent, no route for %s/%s",
+                    message.domain(),
+                    message.path(),
+                )
+
+        except Exception as err:
+            logging_error("Send message failed: %s", err)
+
+    async def request_routes(self):
+        await self.router.request_routes_from_adapters()
+
+    def find_route(self, domain: str, path: str) -> AMQRoute:
+        return self.router.find_route(domain, path)

amqp/rabbitmq/rabbit_mq_consumer.py

@@ -1,143 +0,0 @@
-import logging
-import pika
-from pika.exchange_type import ExchangeType
-
-from amqp.model.model import AMQRoute
-from amqp.rabbitmq.rabbit_mq_producer import RabbitMQProducer
-from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE, DLQ_EXCHANGE, AMQ_RPC_EXCHANGE
-from amqp.router.router_consumer import RouterConsumer
-from amqp.router.utils import sanitize_as_rabbitmq_name
-
-
-class RabbitMQConsumer(RabbitMQProducer):
-    PREFETCH_COUNT = 1
-    DLQ_ARGS = {"x-dead-letter-exchange": DLQ_EXCHANGE}
-
-    def __init__(self, configuration):
-        super().__init__(configuration, RouterConsumer(configuration))
-
-    def cleanup(self):
-        logging.info(" [*] RabbitMQConsumer.cleanup()")
-        self.router.cleanup()  # de-register route(s)
-        if self.router.is_open(self.channel):
-            try:
-                self.channel.close()
-            except (pika.connection.exceptions.AMQPError, pika.connection.exceptions.ChannelClosedByBroker) as e:
-                logging.error(" [E] RabbitMQConsumer.PreDestroy cleanup - connection close error: %s", e)
-
-    def register_routes(self):
-        if self.router.is_open(self.channel):
-            queue_args = self.DLQ_ARGS if self.amq_configuration.dispatch.use_dlq else None
-            route_mapping = self.amq_configuration.amq_adapter.route_mapping
-            logging.info(" [*] RabbitMQConsumer register routes, cfg mapping: [%s]", route_mapping)
-            requested_routes = self.router.unwrap_route_list(route_mapping)
-            for route in requested_routes:
-                queue_name = route.queue() if route.queue() else self.router.get_consume_queue_name()
-                logging.info(" [*] RabbitMQConsumer register route: [%s]", route)
-                self.channel.queue_declare(queue_name, durable=True, exclusive=False,
-                                           auto_delete=False, arguments=queue_args)
-                exchange = route.exchange() if route.exchange() else AMQ_RPC_EXCHANGE
-                self.channel.exchange_declare(exchange, exchange_type=pika.exchange_type.ExchangeType.topic)
-                self.channel.queue_bind(queue_name, exchange, route.key())
-                self.router.register_route(
-                    AMQRoute(
-                        route.component_name(), exchange, queue_name,
-                        sanitize_as_rabbitmq_name(route.key()),
-                        route.timeout(), route.valid_until()
-                    )
-                )
-                logging.info(" [*] AMQ connection initialized for route: %s, listens on: %s/%s",
-                             route, exchange, queue_name)
-        else:
-            logging.warning(" [W] Channel is null, cannot register routes.")
-
-    def register_deliver_callback(self, callback_provider):
-        try:
-            for route in self.router.get_own_routes():
-                self.channel.basic_consume(
-                    queue=route.queue(),
-                    on_message_callback=callback_provider(self.channel),
-                    auto_ack=False
-                )
-                logging.info(" [*] Consumer Waiting for messages on: %s", route.queue())
-        except pika.connection.exceptions.AMQPError as err:
-            logging.error(" [E] Callback registration INCOMPLETE, %s", err)
-
-    def register_rpc_handler(self, rpc_consumer):
-        cm_reply_to_queue = self.router.get_reply_to_queue_name()
-        self.channel.queue_declare(cm_reply_to_queue, durable=True, exclusive=False, auto_delete=False)
-        self.channel.exchange_declare(
-            AMQ_REPLY_TO_EXCHANGE, exchange_type=pika.exchange_type.ExchangeType.direct
-        )
-        self.channel.queue_bind(cm_reply_to_queue, AMQ_REPLY_TO_EXCHANGE, cm_reply_to_queue)
-        self.channel.basic_consume(
-            queue=cm_reply_to_queue,
-            on_message_callback=rpc_consumer,
-            auto_ack=False
-        )
-
-    """
-    def limit_rate_inside_time_window(self, message: AMQMessage) -> int:
-        next_batch_size = BackpressureSubscriber.NO_OP
-        now = time.time_ns() // 1000000
-        queue_size = self.getQueueSize(message)
-
-        # 1. Backpressure? if unprocessed count exceeds threshold, wait for some time
-        if queue_size > THRESHOLD:
-            # TODO - signal queue backpressure explicitly to upstream/metrics
-            # wait: estimated-per-message-processing-time * SQRT(over-the-limit)
-            self.wait_for_milliseconds(
-                (WINDOW / self.request_for_window_duration) * (queue_size - THRESHOLD) ** 0.5)
-            queue_size = self.getQueueSize(message)  # refresh queue size after wait
-
-        # 2. batch complete? determine next batch size and wait till window end. reset window.
-        if self.fulfilled >= self.request_for_window_duration:
-            next_batch_size = self.calculateNextBatchSize(queue_size)
-            self.request_for_window_duration = next_batch_size
-            remaining_time = (self.window_started_at + WINDOW) - now
-            if remaining_time > 0:
-                self.wait_for_milliseconds(remaining_time)
-                queue_size = self.getQueueSize(message)  # refresh queue size after wait
-            self.window_started_at = time.time_ns() // 1000000
-            self.fulfilled = 0
-
-        # 3. Window elapsed? Reset window & recalculate batch size
-        if now >= self.window_started_at + WINDOW:
-            if next_batch_size == BackpressureSubscriber.NO_OP:
-                next_batch_size = self.calculateNextBatchSize(queue_size)
-                self.request_for_window_duration = next_batch_size
-            self.window_started_at = time.time_ns() // 1000000
-            self.fulfilled = 0
-
-        # 4. send the message from the current batch
-        self.send_message(message)
-        self.fulfilled += 1
-
-        # 5. report and setup state for the next call
-        end = time.time_ns() // 1000000
-        self.on_next_timestamp = end
-        return next_batch_size
-
-    def calculateNextBatchSize(self, queue_size: int) -> int:
-        next_batch_size = self.request_for_window_duration
-        if queue_size <= THRESHOLD:
-            # processing proceeds well, can increase the rate
-            next_batch_size += self.subscriber.increaseRate(self.request_for_window_duration)
-            logging.debug(f"INCREASE rate to {next_batch_size}. qSize={queue_size}")
-        else:
-            decrease = self.subscriber.backoff(queue_size - THRESHOLD)
-            if self.request_for_window_duration > decrease:
-                next_batch_size -= decrease
-            logging.debug(f"DECREASE rate to {next_batch_size}. qSize={queue_size}")
-        return next_batch_size
-    """
-    """
-    def getQueueSize(self, message: AMQMessage) -> int:
-        try:
-            route_queue_name = self.router.findRoute(message)
-            queue_name = route_queue_name.queue if route_queue_name else self.TASK_DISPATCH_QUEUE_NAME
-            logger.info(f"Get Queue size for {queue_name}")
-            return max(len(self.awaitingACK), self.channel.message_count(queue_name))
-        except Exception:
-            return len(self.awaitingACK)
-    """

amqp/rabbitmq/rabbit_mq_producer.py

@@ -1,134 +0,0 @@
-import logging
-import time
-
-import pika
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import AMQMessage, AMQRoute
-from amqp.router.router_producer import RouterProducer
-
-
-class RabbitMQProducer:
-    PREFETCH_COUNT = 1
-    THRESHOLD = 1000
-    WINDOW = 5000  # 5 seconds
-
-    def __init__(self, amq_configuration: AMQConfiguration, router: RouterProducer = None):
-        self.channel = None
-        self.connection = None
-        self.request_for_window_duration = None
-        self.amq_configuration = amq_configuration
-        logging.info("*******************************************")
-        logging.info("******** RabbitMQProducer.init() ********** %s - %s - %s",
-                     self.amq_configuration.dispatch.amq_host,
-                     self.amq_configuration.dispatch.use_confirms,
-                     self.WINDOW)
-        logging.info("*******************************************")
-        self.router = router or RouterProducer(amq_configuration)
-
-        # this.meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
-
-        self.setup_amq_channel()
-        self.router.init_routing_paths(self.channel)
-
-        #  self.backpressure_per_exchange = defaultdict(lambda: Gauge(
-        #    f"amqp.adapter.exchange.{exchange}", registry=CollectorRegistry()))
-
-        self.router.set_route_added_notifier(
-            lambda route: logging.info(" [DBG] ---------- RouteAdded, setting up metrics: amqp.adapter.exchange.%s",
-                                       route.exchange))
-
-        self.request_for_window_duration = self.THRESHOLD // 2
-        self.window_started_at = time.time()
-        self.on_next_timestamp = self.window_started_at
-        self.fulfilled = 0
-
-    def cleanup(self):
-        if self.connection:
-            try:
-                self.connection.close()
-            except Exception as e:
-                logging.error("RabbitMQConsumer.PreDestroy cleanup - connection close error: %s", e)
-
-    def setup_amq_channel(self):
-        """
-        Set up the RabbitMQ connection
-        :return: nothing, it applies changes to instance variables
-        """
-        factory = pika.ConnectionParameters(
-            host=self.amq_configuration.dispatch.amq_host,
-            username=self.amq_configuration.dispatch.rabbit_mq_user,
-            password=self.amq_configuration.dispatch.rabbit_mq_password
-        )
-        self.connection = pika.BlockingConnection(factory)
-        logging.info("RabbitMQProducer.setupAMQ, got connection, open=%s", self.connection.is_open)
-        self.connection.add_on_connection_blocked_callback(self.blocked_listener)
-        self.connection.add_on_connection_unblocked_callback(self.unblocked_listener)
-        self.channel = self.connection.channel()
-        self.channel.basic_qos(prefetch_count=self.PREFETCH_COUNT)
-        if self.amq_configuration.dispatch.use_confirms():
-            self.setup_local_ack()
-        self.channel.add_on_return_callback(self.return_listener)
-
-    def send_message(self, message: AMQMessage):
-        """
-        The API routine that sends a datagram message (AMQMessage) to the destination. The Destination
-        is determined from the AMQMessage metadata section, using `domain` and `path` as key.
-        :param message: message to send
-        :return: nothing (void) as submitting message to RabbitMQ exchange does not return anything
-         meaningful. There will be asynch ACK later that will confirm the message was sent to RMQ.
-        """
-        try:
-            route = self.router.find_route_by_message(message)
-            if route:
-                if route.timeout < 0:
-                    # No response expected
-                    self.channel.basic_publish(
-                        route.exchange,
-                        self.router.get_routing_key(message),  # context path is a routing key
-                        properties=pika.BasicProperties(delivery_mode=2),
-                        body=AMQMessageFactory.serialize(message)
-                    )
-                    logging.info(f" [x] basicPublish (no-reply), messageId: {message.id}, to: {route}")
-                else:
-                    # This is RPC call, there should be response
-                    props = pika.BasicProperties(
-                        correlation_id=message.id,
-                        reply_to=self.router.get_reply_to_queue_name()
-                    )
-                    self.channel.basic_publish(
-                        route.exchange,
-                        self.router.get_routing_key(message),  # context path is a routing key
-                        properties=props,
-                        body=AMQMessageFactory.serialize(message)
-                    )
-                    logging.info(" [x] basicPublish (RPC call), messageId: %s, to: %s",
-                                 message.id.as_string(), route.as_string())
-            else:
-                logging.warning("Message not sent, no route for {}/{}", message.domain, message.path)
-
-        except Exception as err:
-            logging.error("Send message failed: %s", err)
-
-    def request_routes(self):
-        self.router.request_routes_from_adapters()
-
-    def setup_local_ack(self):
-        self.channel.confirm_delivery()
-
-    def find_route(self, domain: str, path: str) -> AMQRoute:
-        return self.router.find_route(domain, path)
-
-    def return_listener(self, return_message):
-        logging.error(
-            f" [E] ****** (NOT DELIVERED) ID={return_message.properties.message_id}, "
-            f"Exchg=[{return_message.exchange}], Key=[{return_message.routing_key}], "
-            f"replyTxt={return_message.reply_text}, length={len(return_message.body) if return_message.body else 0}"
-        )
-
-    def blocked_listener(self, reason: str):
-        logging.error(f" [E] CHANNEL BLOCKED, reason={reason}")
-
-    def unblocked_listener(self):
-        logging.info(" [*] CHANNEL UNBLOCKED")

amqp/router/route_database.py

@@ -1,7 +1,7 @@
-import logging
 import re
 from typing import Set
 
+from amqp.adapter.logging_utils import logging_info
 from amqp.model.model import AMQRoute
 from amqp.router.utils import sanitize_as_rabbitmq_name
 
@@ -14,14 +14,30 @@ class RouteDatabase:
         return self.routes
 
     def pattern(self, routing_key: str) -> re.Pattern:
-        regex = r"\.".join(
-            [
-                r".*" if token == "" else r"#" if token == "#" else token
-                for token in routing_key.split(".")
-            ]
-        )
-        if routing_key.endswith("."):
-            regex += r"\."
+        breaker = False
+        tokens = routing_key.split(".")
+        counter = len(tokens)
+        regex_parts = []
+
+        for token in tokens:
+            counter -= 1
+            if not breaker:
+                if token == "":
+                    res = "[^\\.]*"
+                elif token == "#":
+                    breaker = True
+                    res = ".*$"
+                else:
+                    res = token
+                if counter > 0 and not breaker:
+                    res += "\\."
+                regex_parts.append(res)
+            else:
+                regex_parts.append("")
+
+        regex = "".join(filter(lambda x: x != "", regex_parts))
+        if not routing_key:
+            return re.compile(".*")
         return re.compile(regex)
 
     def matches(self, routing_key_from_route: str, routing_key_from_message: str) -> bool:
@@ -33,7 +49,7 @@ class RouteDatabase:
 
     def find_route(self, domain: str, path: str) -> AMQRoute | None:
         routing_key = sanitize_as_rabbitmq_name(f"{domain}.{path}")
-        logging.info(f" [DBG] findRoute key:{routing_key} in:{self.wrap_routes()}")
+        logging_info(f"findRoute key:{routing_key} in:{self.wrap_routes()}")
         for route in self.routes:
             if self.matches(route.key, routing_key):
                 return route

amqp/router/router_base.py

@@ -1,8 +1,8 @@
-import logging
 from typing import Callable, List, Set
 
-from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
-from amqp.model.model import AMQRoute, AMQMessage
+from amqp.adapter.amq_route_factory import NULL_ROUTE, AMQRouteFactory
+from amqp.adapter.logging_utils import logging_debug, logging_error, logging_info
+from amqp.model.model import AMQRoute, DataMessage
 from amqp.router.route_database import RouteDatabase
 from amqp.router.utils import sanitize_as_rabbitmq_name
 
@@ -26,8 +26,8 @@ class RouterBase:
         self.route_added_notifier: Callable[[AMQRoute], None] | None = None
         self.route_removed_notifier: Callable[[AMQRoute], None] | None = None
 
-    def get_routing_key(self, message: AMQMessage) -> str:
-        return sanitize_as_rabbitmq_name(f"{message.domain}.{message.path}")
+    def get_routing_key(self, message: DataMessage) -> str:
+        return sanitize_as_rabbitmq_name(f"{message.domain()}.{message.path()}")
 
     def wrap_routes(self) -> str:
         return self.route_database.wrap_routes()
@@ -36,7 +36,7 @@ class RouterBase:
         return ",".join(str(route) for route in self.own_routes)
 
     def unwrap_route_list(self, route_string: str) -> List[AMQRoute]:
-        logging.info(" [DBG] RouterMaster.unwrapRoutes, route(s): %s", route_string)
+        logging_debug("RouterMaster.unwrapRoutes, route(s): %s", route_string)
         routes = [AMQRouteFactory.from_string(route_str) for route_str in route_string.split(",")]
         return [route for route in routes if route != NULL_ROUTE]
 
@@ -46,26 +46,30 @@ class RouterBase:
     def find_route(self, domain: str, path: str) -> AMQRoute | None:
         return self.route_database.find_route(domain, path)
 
-    def find_route_by_message(self, message: AMQMessage) -> AMQRoute | None:
-        return self.route_database.find_route(message.domain, message.path)
+    def find_route_by_message(self, message: DataMessage) -> AMQRoute | None:
+        return self.route_database.find_route(message.domain(), message.path())
 
     def add_routes(self, message: str) -> None:
         try:
             amq_route_list = self.unwrap_route_list(message)
-            logging.info(" [DBG] RouterMaster.addRoute [%d]: %s", len(amq_route_list), message)
+            logging_info(" [DBG] RouterMaster.addRoute [%d]: %s", len(amq_route_list), message)
             for route in amq_route_list:
-                logging.info(" [*] RouterMaster.addRoute, adding route: %s", route)
+                logging_info("RouterMaster.addRoute, adding route: %s", route)
                 if not route.exchange and not route.queue and not route.key:
-                    logging.error("Cannot setup AMQ Exchange for '%s', at least one of "
-                                  "Exchange, Queue and/or Key must be present in the routing string. "
-                                  "Expected format is '%s'", route, AMQRoute.FORMAT)
+                    logging_error(
+                        "Cannot setup AMQ Exchange for '%s', at least one of "
+                        "Exchange, Queue and/or Key must be present in the routing string. "
+                        "Expected format is '%s'",
+                        route,
+                        AMQRoute.FORMAT,
+                    )
                 else:
                     if route.exchange != DLQ_EXCHANGE:
                         is_new_route = self.route_database.add_route(route)
                         if is_new_route and self.route_added_notifier:
                             self.route_added_notifier(route)
         except Exception as err:
-            logging.error(" [E] add_routes error: %s", err)
+            logging_error("add_routes error: %s", err)
 
     def set_route_added_notifier(self, callback: Callable[[AMQRoute], None]):
         self.route_added_notifier = callback
@@ -74,9 +78,9 @@ class RouterBase:
         try:
             to_remove = [AMQRouteFactory.from_string(route_str) for route_str in message.split(",")]
             removed_count = sum(1 for route in to_remove if self.remove_route(route))
-            print(f" [*] RouterMaster.removeRoutes(): {message}, removed={removed_count}")
+            logging_info(f"RouterMaster.removeRoutes(): {message}, removed={removed_count}")
         except IOError as err:
-            print(f" [E] Can't remove route(s): {err}")
+            logging_error(f"Can't remove route(s): {err}")
 
     def remove_route(self, route: AMQRoute) -> bool:
         is_route_removed = self.route_database.remove_route(route)

amqp/router/router_consumer.py

@@ -1,143 +1,200 @@
-import logging
 from typing import Set
 
-from pika import spec
-from pika.adapters.blocking_connection import BlockingChannel
+from aio_pika.abc import (
+    AbstractIncomingMessage,
+    AbstractRobustChannel,
+    AbstractRobustQueue,
+)
 
 from amqp.adapter.amq_route_factory import AMQRouteFactory
+from amqp.adapter.logging_utils import (
+    logging_debug,
+    logging_error,
+    logging_info,
+    logging_warning,
+)
 from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import CleverMicroServiceMessage, ServiceMessageType, AMQRoute
-from amqp.router.router_base import CM_C_REQ_QUEUE, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, ANY_RECIPIENT
+from amqp.model.model import AMQRoute, ServiceMessage
+from amqp.model.service_message_type import ServiceMessageType
+from amqp.router.router_base import (
+    ANY_RECIPIENT,
+    CM_C_REQ_QUEUE,
+    CM_REQUEST_EXCHANGE,
+    CM_RESPONSE_EXCHANGE,
+)
 from amqp.router.router_producer import RouterProducer
 
 
 class RouterConsumer(RouterProducer):
     """
-     * Contains Consumer logic. The logic here:
-     * Sets a listener on CM_C_REQ_QUEUE queue to receive routing data requests
-     * Upon receipt of routing data request, responds on CM_RESPONSE_EXCHANGE with local routing data.
-     * It also proactively pushes the local routing data to CM_RESPONSE_EXCHANGE on startup to
-     * notify the other (excluding self) Producers about its presence.
-     *
-     * Consumer is also Producer (Consumer sits alongside the CM Service, and anytime the Service needs
-     * to communicate with other Service, it can do so via the Adapter). Therefore, the Producer
-     * capability is also required.
+    * Contains Consumer logic. The logic here:
+    * Sets a listener on CM_C_REQ_QUEUE queue to receive routing data requests
+    * Upon receipt of routing data request, responds on CM_RESPONSE_EXCHANGE with local routing data.
+    * It also proactively pushes the local routing data to CM_RESPONSE_EXCHANGE on startup to
+    * notify the other (excluding self) Producers about its presence.
+    *
+    * Consumer is also Producer (Consumer sits alongside the CM Service, and anytime the Service needs
+    * to communicate with other Service, it can do so via the Adapter). Therefore, the Producer
+    * capability is also required.
     """
 
     def __init__(self, configuration: AMQConfiguration):
         super().__init__(configuration)
         self.amqRouteFactory = AMQRouteFactory()
-        logging.info(" [*] RouterConsumer.<init>")
+        logging_info("RouterConsumer.<init>")
 
-    def init_routing_paths_consumer(self, _channel: BlockingChannel):
+    async def init_routing_paths_consumer(self, _channel: AbstractRobustChannel):
         """
-         * Initialize router according to Consumer mode.
+        * Initialize router according to Consumer mode.
         """
-
-        self.init_routing_paths(_channel)
-        logging.info(" [*] RouterConsumer.initConsumerRouter, channel open::", self.is_open(_channel))
+        logging_info(
+            "RouterConsumer.initConsumerRouter, channel open: %s",
+            self.is_open(_channel),
+        )
         if self.is_open(_channel):
             try:
-                #  1. declare a Queue that gives the RoutingDAta requests
-                cm_request_queue: str = self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
-                self.channel.queue_declare(cm_request_queue, durable=True, exclusive=False, auto_delete=False)
-                # 1a. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
-                self.channel.queue_bind(queue=cm_request_queue, exchange=CM_REQUEST_EXCHANGE, routing_key="")
+                #  1. declare a Queue to receive the RoutingData requests
+                _cm_request_queue: str = (
+                    self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
+                )
+                _queue: AbstractRobustQueue = await self.channel.declare_queue(
+                    name=_cm_request_queue,
+                    durable=True,
+                    exclusive=False,
+                    auto_delete=False,
+                )
+                # 1a. bind queue to CM_REQUEST_EXCHANGE w/o any conditions (empty routing key)
+                await _queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="")
                 # 1b. Service adapter mode - listens on Request queue and replies to Response queue
-                logging.info(" [*] RouterConsumer listens for RouteDataRequests on:", cm_request_queue)
-                self.channel.basic_consume(queue=cm_request_queue,
-                                           auto_ack=False,
-                                           on_message_callback=self.handle_routing_data_request_message)
+                _c_tag = await _queue.consume(
+                    callback=self.handle_routing_data_request_message, no_ack=False
+                )
+                logging_info(
+                    " [%s] RouterConsumer listens for RouteDataRequests on: %s",
+                    _c_tag,
+                    _cm_request_queue,
+                )
             except Exception as ioe:
-                logging.error(" [E] RouterConsumer FATAL: cannot initialize, %s", ioe)
+                logging_error("RouterConsumer FATAL: cannot initialize, %s", ioe)
         else:
-            logging.error(" [E] RouterConsumer FATAL: cannot initialize, RabbitMQ channel is not open.")
+            logging_error("RouterConsumer FATAL: cannot initialize, RabbitMQ channel is not open.")
 
     """
      * ServiceMessageType.ROUTING_DATA_REQUEST  handler  listening on queue 'CleverMicroRequest'
     """
 
-    def handle_routing_data_request_message(self,
-                                            ch: BlockingChannel,
-                                            method: spec.Basic.Deliver,
-                                            properties: spec.BasicProperties,
-                                            body: bytes):
+    async def handle_routing_data_request_message(self, message: AbstractIncomingMessage):
         # some debug statements, TODO - remove for prod release
-        delivery_tag = method.delivery_tag
-        debug = "Delivery.Properties = " + properties.__str__()
-        logging.info(" [DBG] ******[%] (ROUTING_DATA_REQ) tag: %s, env: %s, props: %s",
-                     delivery_tag, method.consumer_tag, method.get_properties(), debug)
+        delivery_tag = message.delivery_tag
+        debug = "Delivery.Properties = " + message.properties.__str__()
+        logging_debug(
+            " ******[%s] (ROUTING_DATA_REQ) tag: %s, env: %s, props: %s",
+            delivery_tag,
+            message.consumer_tag,
+            message.properties,
+            debug,
+        )
 
-        self.channel.basic_ack(delivery_tag, False)
+        await message.ack(False)
 
-        message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(body)
+        cm_message: ServiceMessage = self.service_message_factory.from_bytes(message.body)
         route_mapping: str = self.wrap_own_routes()
-        logging.info(" [DBG] ******[%s] (ROUTING_DATA_REQ) msg=%, routeMap=[{]",
-                     delivery_tag, self.service_message_factory.to_string(message), route_mapping)
+        logging_debug(
+            "******[%s] (ROUTING_DATA_REQ) msg=%s, routeMap=[%s]",
+            delivery_tag,
+            self.service_message_factory.to_string(cm_message),
+            route_mapping,
+        )
         # Reply only if self adapter has own routeMapping (e.g. API-gateway-side adapter has none)
-        if message.is_valid() and message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST and route_mapping:
-            try:
-                service_message: CleverMicroServiceMessage = self.service_message_factory.of(
-                    ServiceMessageType.ROUTING_DATA, route_mapping, message.reply_to
+        if (
+            cm_message.is_valid()
+            and cm_message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST
+            and route_mapping
+        ):
+            if cm_message.reply_to != self.amq_configuration.amq_adapter.service_name:
+                try:
+                    service_message: ServiceMessage = self.service_message_factory.of(
+                        ServiceMessageType.ROUTING_DATA,
+                        route_mapping,
+                        cm_message.reply_to,
+                    )
+                    await self.publish_service_message(service_message, self.cm_response_exchange)
+                except Exception as err:
+                    logging_error(
+                        "******[%s] (ROUTING_DATA_REQ) Response to queue: %s failed: %s",
+                        delivery_tag,
+                        CM_RESPONSE_EXCHANGE,
+                        err,
+                    )
+            else:
+                logging_info(
+                    "******[%s] (ROUTING_DATA_REQ) [%s] Ignore request originated by ourselves.",
+                    delivery_tag,
+                    str(message.body),
                 )
-                self.publish_service_message(service_message, CM_RESPONSE_EXCHANGE)
-            except Exception as err:
-                logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) Response to queue: %s failed: %s",
-                              delivery_tag, CM_RESPONSE_EXCHANGE, err)
-
         else:
-            logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) [%s] produces invalid or no CM msg type.",
-                          delivery_tag, str(body))
+            logging_error(
+                "******[%s] (ROUTING_DATA_REQ) [%s] produces invalid or no CM msg type.",
+                delivery_tag,
+                str(message.body),
+            )
 
-    def register_route(self, route: AMQRoute):
+    async def register_route(self, route: AMQRoute):
         """
-         * Called in Consumer (Service Adapter) context. Registers the routing data for self Service
-         * with the Master router by sending own routing data to Master via CleverMicroResponse queue.
-         * :param route: route to register with Master
+        * Called in Consumer (Service Adapter) context. Registers the routing data for self Service
+        * with the Master router by sending own routing data to Master via CleverMicroResponse queue.
+        * :param route: route to register with Master
         """
         try:
             if self.is_open(self.channel) and route.queue:
-                serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
+                serviceMessage: ServiceMessage = self.service_message_factory.of(
                     ServiceMessageType.ROUTING_DATA, route.as_string(), ANY_RECIPIENT
                 )
-                self.publish_service_message(serviceMessage, CM_RESPONSE_EXCHANGE)
+                await self.publish_service_message(serviceMessage, self.cm_response_exchange)
                 self.add_own_route(route)
-                logging.info(" [DBG] RouterConsumer registered Route::", route)
+                logging_info("RouterConsumer registered Route: %s", route)
             else:
-                logging.warning(" [W] RouterConsumer couldn't register route data: %s channel isn't open.",
-                                CM_RESPONSE_EXCHANGE)
+                logging_warning(
+                    "RouterConsumer couldn't register route data: %s channel isn't open.",
+                    CM_RESPONSE_EXCHANGE,
+                )
         except Exception as ioe:
-            logging.error(" [E] RouterConsumer failed register route data: %s", ioe)
+            logging_error("RouterConsumer failed register route data: %s", ioe)
 
-    def remove_route_on_cleanup(self, route: AMQRoute) -> bool:
+    async def remove_route_on_cleanup(self, route: AMQRoute) -> bool:
         """
-         * Unregister / remove route from the list
-         * :param route: route to remove
-         * :return result of remove operation
+        * Unregister / remove route from the list
+        * :param route: route to remove
+        * :return result of remove operation
         """
-        logging.info(" [DBG] RouterConsumer.removeRoute, route::", route.as_string())
+        logging_info("RouterConsumer.removeRoute, route: %s", route.as_string())
         try:
-            service_message: CleverMicroServiceMessage = self.service_message_factory.of(
-                ServiceMessageType.ROUTING_DATA_REMOVE,
-                route.as_string(),
-                ANY_RECIPIENT
+            service_message: ServiceMessage = self.service_message_factory.of(
+                ServiceMessageType.ROUTING_DATA_REMOVE, route.as_string(), ANY_RECIPIENT
             )
-            if not self.publish_service_message(service_message, CM_RESPONSE_EXCHANGE):
-                logging.warning(" [W] RouterConsumer couldn't remove current route: %s, channel not open.",
-                                route)
+            if not await self.publish_service_message(service_message, self.cm_response_exchange):
+                logging_warning(
+                    "RouterConsumer couldn't remove current route: %s, channel not open.",
+                    route,
+                )
 
         except Exception as ioe:
-            logging.error(" [E] RouterConsumer failed remove current route: %s", ioe)
+            logging_error("RouterConsumer failed remove current route: %s", ioe)
 
         result: bool = self.remove_route(route)
         return result
 
     def cleanup(self):
         """
-         * Cleanup - de-register routes with master.
+        * Cleanup - de-register routes with master.
         """
         routes: Set[AMQRoute] = self.get_routes()
-        logging.info(" [DBG] RouterConsumer.cleanup() [%d]: %s", len(routes), self.wrap_routes())
-        not_removed = sum(1 for r in (self.remove_route_on_cleanup(route) for route in routes) if not r)
+        logging_info("RouterConsumer.cleanup() [%d]: %s", len(routes), self.wrap_routes())
+        not_removed = sum(
+            1 for r in (self.remove_route_on_cleanup(route) for route in routes) if not r
+        )
         if not_removed > 0:
-            logging.info(" [W] %d route(s) not removed. self may lead to failed delivery on self route", not_removed)
+            logging_warning(
+                "%d route(s) not removed. self may lead to failed delivery on self route",
+                not_removed,
+            )

amqp/router/router_producer.py

@@ -1,190 +1,225 @@
 """
- * Maps the request to RabbitMQ exchange. Handles the service discovery logic.
- *
- * Router can be either in 'Master' or 'Service (Consumer)' modes. 'Master' mode is the outward
- * facing side that receives the requests and routes it to queue associated with requested service.
- * 'Service' mode is consumer side for given service, which receives the message from AMQ and
- * forwards it to the application.
- """
-import logging
-import os
-import pika
-from pika import BasicProperties, spec
-from pika.adapters.blocking_connection import BlockingChannel
+* Maps the request to RabbitMQ exchange. Handles the service discovery logic.
+*
+* Router can be either in 'Master' or 'Service (Consumer)' modes. 'Master' mode is the outward
+* facing side that receives the requests and routes it to queue associated with requested service.
+* 'Service' mode is consumer side for given service, which receives the message from AMQ and
+* forwards it to the application.
+"""
+
+from aio_pika import Message
+from aio_pika.abc import (
+    AbstractIncomingMessage,
+    AbstractRobustChannel,
+    AbstractRobustConnection,
+    AbstractRobustExchange,
+    AbstractRobustQueue,
+)
 from pika.exchange_type import ExchangeType
 
-from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
+from amqp.adapter.logging_utils import (
+    logging_debug,
+    logging_error,
+    logging_info,
+    logging_warning,
+)
+from amqp.adapter.service_message_factory import ServiceMessageFactory
 from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import CleverMicroServiceMessage, ServiceMessageType
-from amqp.router.router_base import RouterBase, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, CM_C_AMQ_QUEUE, \
-    CM_P_RESP_QUEUE, CM_P_REPLY_TO, ANY_RECIPIENT
+from amqp.model.model import ServiceMessage
+from amqp.model.service_message_type import ServiceMessageType
+from amqp.router.router_base import (
+    ANY_RECIPIENT,
+    CM_C_AMQ_QUEUE,
+    CM_P_REPLY_TO,
+    CM_P_RESP_QUEUE,
+    CM_REQUEST_EXCHANGE,
+    CM_RESPONSE_EXCHANGE,
+    RouterBase,
+)
 
 
 class RouterProducer(RouterBase):
 
     def __init__(self, configuration: AMQConfiguration):
         """
-         * Initialize router - need to supply the config values and RabbitMQ channel.
-         * :param configuration:  adapter's configuration reference.
-         """
-        super().__init__()
-        self.connection = None
-        self.service_message_factory = CleverMicroServiceMessageFactory(configuration.amq_adapter.service_name)
-        self.amq_configuration: AMQConfiguration = configuration
-        self.channel: pika.adapters.blocking_connection.BlockingChannel | None = None
-
-        logging.info(f" [*] RouterProducer.<init>: %s, route: %s",
-                     self.amq_configuration.amq_adapter.service_name,
-                     self.amq_configuration.amq_adapter.route_mapping)
-
-    def amq_consumer(self):
-        credentials = pika.PlainCredentials(
-            os.environ['RABBIT_MQ_USER'], os.environ['RABBIT_MQ_PASSWORD']
-        )
-        self.connection = pika.BlockingConnection(
-            pika.ConnectionParameters(host='localhost', credentials=credentials)
-        )
-        self.channel = self.connection.channel()
-        self.channel.start_consuming()
-
-    def init_routing_paths(self, channel: pika.adapters.blocking_connection.BlockingChannel | None):
+        * Initialize router - need to supply the config values and RabbitMQ channel.
+        * :param configuration:  adapter's configuration reference.
+        """
+        super().__init__()
+        self.connection: AbstractRobustConnection | None = None
+        self.service_message_factory = ServiceMessageFactory(configuration.amq_adapter.service_name)
+        self.amq_configuration: AMQConfiguration = configuration
+        self.channel: AbstractRobustChannel | None = None
+        self.cm_request_exchange: AbstractRobustExchange | None = None
+        self.cm_response_exchange: AbstractRobustExchange | None = None
+
+        logging_info(
+            "RouterProducer.<init>: %s, route: %s",
+            self.amq_configuration.amq_adapter.service_name,
+            self.amq_configuration.amq_adapter.route_mapping,
+        )
+
+    async def init_internal_routing_paths(self, channel: AbstractRobustChannel | None):
+        """
+        * Initialize internal routes for ServiceMessages.
+        * The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
+        * created to enable the service discovery.
+        * The Producer mode sends out routing data request on CleverMicroRequest and listens
+        * on CleverMicroResponse for any service responding with its routing data.
         """
-         * Initialize router according to Producer mode.
-         * The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
-         * created to enable the service discovery.
-         * The Producer mode sends out routing data request on CleverMicroRequest and listens
-         * on CleverMicroResponse for any service responding with its routing data.
-         """
 
         self.channel = channel
-        logging.info(" [*] RouterProducer.initRoutingPaths, channel open: %s", self.is_open(self.channel))
+        logging_info(
+            "RouterProducer.initRoutingPaths, channel open: %s",
+            self.is_open(self.channel),
+        )
         if self.is_open(self.channel):
             try:
                 # **** set up the service discovery internal exchanges ****
                 # 1. declare RequestExchange to where to publish the RoutingData requests
-                self.channel.exchange_declare(exchange=CM_REQUEST_EXCHANGE, exchange_type=ExchangeType.fanout)
+                self.cm_request_exchange = await self.channel.declare_exchange(
+                    name=CM_REQUEST_EXCHANGE, type=ExchangeType.fanout
+                )
                 #
                 # 2. declare ResponseExchange where to publish current RoutingData (response to
                 #    a RoutingData request)
-                self.channel.exchange_declare(exchange=CM_RESPONSE_EXCHANGE, exchange_type=ExchangeType.fanout)
+                self.cm_response_exchange = await self.channel.declare_exchange(
+                    name=CM_RESPONSE_EXCHANGE, type=ExchangeType.fanout
+                )
                 # 2a. declare a Response queue for Routing Data replies from Services
-                cm_response_queue: str = self.get_response_queue_name()
-                self.channel.queue_declare(cm_response_queue, durable=True, exclusive=False, auto_delete=False,
-                                           arguments={})
+                _cm_response_queue: str = self.get_response_queue_name()
+                _response_queue: AbstractRobustQueue = await self.channel.declare_queue(
+                    name=_cm_response_queue,
+                    durable=True,
+                    exclusive=False,
+                    auto_delete=False,
+                    arguments={},
+                )
                 # 2b. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
-                self.channel.queue_bind(queue=cm_response_queue, exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
+                await _response_queue.bind(exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
                 # 2c. listen for incoming RoutingData messages
-                self.channel.basic_consume(
-                    queue=cm_response_queue,
-                    auto_ack=False,
-                    on_message_callback=self.handle_routing_data_message)
-                self.channel.start_consuming()
-                logging.info(" [*] Routing master listens for Route updates on %s", cm_response_queue)
+                _c_tag = await _response_queue.consume(
+                    callback=self.handle_routing_data_message, no_ack=False
+                )
+                logging_info(
+                    " [%s] Routing master listens for Route updates on %s",
+                    _c_tag,
+                    _cm_response_queue,
+                )
             except Exception as ioe:
-                logging.error(" [E] RouterProducer FATAL: cannot initialize, %s", ioe)
+                logging_error("RouterProducer FATAL: cannot initialize, %s", ioe)
 
         else:
-            logging.error(" [E] RouterProducer FATAL: cannot initialize, RabbitMQ channel is not open.")
+            logging_error("RouterProducer FATAL: cannot initialize, RabbitMQ channel is not open.")
 
-    def handle_routing_data_message(self,
-                                    ch: BlockingChannel,
-                                    method: spec.Basic.Deliver,
-                                    properties: spec.BasicProperties,
-                                    body: bytes):
+    async def handle_routing_data_message(self, message: AbstractIncomingMessage):
         """
-         * Handler for CLEVER_MICRO_RESPONSE queue messages. Used in Master mode to handle receipt of
-         * routing data, invokes addRoute or removeRoute depending on the type of the message.
-         """
-        print(f" [x] Received {body}, m={method}, p={properties}, ch={ch}")
+        * Handler for CLEVER_MICRO_RESPONSE queue messages. Used in Master mode to handle receipt of
+        * routing data, invokes addRoute or removeRoute depending on the type of the message.
         """
-        logging.info(" [DBG] ******[%s] ROUTING_DATA %s >>> %s",
-              delivery.getEnvelope().getDeliveryTag(),
-              delivery.getEnvelope().toString(), str(delivery.getBody()))
-              """
-        # ACK message now so that it is not being redelivered
-        self.channel.basic_ack(method.delivery_tag, False)
-        message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(body)
+        logging_info(
+            "[%s] Received ROUTING DATA: [%s], msgId=%s",
+            message.delivery_tag,
+            message.body,
+            message.message_id,
+        )
+        await message.ack(multiple=False)
+        cm_message: ServiceMessage = self.service_message_factory.from_bytes(message.body)
         # ignore the message if it comes from ourselves (even if from different instance)
-        logging.info(" [DBG] ******[%s] ROUTING_DATA, msg=%s, routeMap=[{] sn={, replyTo={",
-                     method.delivery_tag,
-                     self.service_message_factory.to_string(message),
-                     self.amq_configuration.amq_adapter.route_mapping,
-                     self.amq_configuration.amq_adapter.service_name,
-                     message.reply_to)
-        if message.is_valid() and not self.amq_configuration.amq_adapter.service_name == message.reply_to:
-            if message.message_type == ServiceMessageType.ROUTING_DATA:
-                self.add_routes(message.message)
-            if message.message_type == ServiceMessageType.ROUTING_DATA_REMOVE:
-                self.remove_routes(message.message)
+        logging_debug(
+            "******[%s] ROUTING_DATA, msg=%s, routeMap=[%s] sn=%s, replyTo=%s",
+            message.delivery_tag,
+            self.service_message_factory.to_string(cm_message),
+            self.amq_configuration.amq_adapter.route_mapping,
+            self.amq_configuration.amq_adapter.service_name,
+            cm_message.reply_to,
+        )
+        if (
+            cm_message.is_valid()
+            and not self.amq_configuration.amq_adapter.service_name == cm_message.reply_to
+        ):
+            if cm_message.message_type == ServiceMessageType.ROUTING_DATA:
+                self.add_routes(cm_message.message)
+            if cm_message.message_type == ServiceMessageType.ROUTING_DATA_REMOVE:
+                self.remove_routes(cm_message.message)
 
-    def request_routes_from_adapters(self):
+    async def request_routes_from_adapters(self):
         """
-         * Send a broadcast message prompting all listening adapters to reply with own routing data.
-         """
-        logging.info(" [*] RouterProducer.requestRoutesFromAdapters, queue name: %s", CM_REQUEST_EXCHANGE)
+        * Send a broadcast message prompting all listening adapters to reply with own routing data.
+        """
+        logging_info(
+            "RouterProducer.requestRoutesFromAdapters, queue name: %s",
+            CM_REQUEST_EXCHANGE,
+        )
         try:
-            serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
+            serviceMessage: ServiceMessage = self.service_message_factory.of(
                 ServiceMessageType.ROUTING_DATA_REQUEST, "", ANY_RECIPIENT
             )
-            if not self.publish_service_message(serviceMessage, CM_REQUEST_EXCHANGE):
-                logging.warning(" [E] RouterProducer could not request route data: %s channel is not open.",
-                                CM_REQUEST_EXCHANGE)
+            if not await self.publish_service_message(serviceMessage, self.cm_request_exchange):
+                logging_warning(
+                    "RouterProducer could not request route data: %s channel is not open.",
+                    CM_REQUEST_EXCHANGE,
+                )
 
         except Exception as ioe:
-            logging.error(" [E] RouterProducer failed request routing data: %s", ioe)
+            logging_error("RouterProducer failed request routing data: %s", ioe)
 
-    BASIC: BasicProperties = BasicProperties(content_type="application/octet-stream",
-                                             content_encoding=None,
-                                             headers=None,
-                                             delivery_mode=1,
-                                             priority=0,
-                                             correlation_id=None)
-
-    def publish_service_message(self, message: CleverMicroServiceMessage, exchange_name: str) -> bool:
+    async def publish_service_message(
+        self, message: ServiceMessage, exchange: AbstractRobustExchange
+    ) -> bool:
+        """
+        * Publishes a service message to the service queue and logs success log entry.
+        *
+        * :param message:   Service Message to be sent
+        * :param exchange: target exchange name where to publish the message
+        * :return True if channel is open, False otherwise
+        * @throws IOException if something else goes wrong
         """
-         * Publishes a service message to the service queue and logs success log entry.
-         *
-         * :param message:   Service Message to be sent
-         * :param exchangeName: target exchange name where to publish the message
-         * :param queueName target: queue name (Optional, purpose is to get the queue size only)
-         * :return True if channel is open, False otherwise
-         * @throws IOException if something else goes wrong
-         """
         if self.is_open(self.channel):
             binary_content: bytes = self.service_message_factory.serialize(message)
-            self.channel.basic_publish(exchange=exchange_name,
-                                       routing_key="",  # empty routing key
-                                       properties=self.BASIC,
-                                       body=binary_content)
-            logging.info(" [x] Service Message Sent to %s, msg: %s", exchange_name, str(binary_content))
+            pika_message: Message = Message(
+                body=binary_content,
+                content_encoding="utf-8",
+                delivery_mode=2,
+                content_type="application/octet-stream",
+                headers=None,
+                priority=0,
+                correlation_id=None,
+            )
+            await exchange.publish(message=pika_message, routing_key="")
+            logging_info(
+                "Service Message Published to %s, msg: %s",
+                exchange.name,
+                str(binary_content),
+            )
             return True
         return False
 
     def get_reply_to_queue_name(self) -> str:
         """
-         * self should create a unique name for self process and also when there's several instances.
-         * @return unique reply-to queue name.
-         """
+        * self should create a unique name for self process and also when there's several instances.
+        * @return unique reply-to queue name.
+        """
         return self.amq_configuration.amq_adapter.adapter_prefix() + CM_P_REPLY_TO
 
     def get_response_queue_name(self) -> str:
         """
-         * self should create a unique name for self process and also when there's several instances.
-         * @return unique reply-to queue name.
-         """
+        * self should create a unique name for self process and also when there's several instances.
+        * @return unique reply-to queue name.
+        """
         return self.amq_configuration.amq_adapter.adapter_prefix() + CM_P_RESP_QUEUE
 
     def get_consume_queue_name(self) -> str:
         """
-         * self should create a unique name for self process and also when there's several instances.
-         * @return unique reply-to queue name.
-         """
+        * self should create a unique name for self process and also when there's several instances.
+        * @return unique reply-to queue name.
+        """
         return self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_AMQ_QUEUE
 
-    def is_open(self, _channel: pika.adapters.blocking_connection.BlockingChannel) -> bool:
+    def is_open(self, _channel: AbstractRobustChannel) -> bool:
         """
-         * Test if the channel is usable (opened).
-         * :param _channel: the channel
-         * :return True if the channel is opened.
-         """
-        return _channel is not None and _channel.is_open
+        * Test if the channel is usable (opened).
+        * :param _channel: the channel
+        * :return True if the channel is opened.
+        """
+        return _channel is not None and not _channel.is_closed

amqp/router/serializer.py

@@ -1,61 +0,0 @@
-import struct
-from io import BytesIO
-from typing import List, Union, Dict
-
-
-def long_to_bytes(x: int) -> bytes:
-    return struct.pack('q', x)
-
-
-def bytes_to_long(b: bytes) -> int:
-    return struct.unpack('q', b)[0]
-
-
-def read_long(bis: BytesIO) -> int:
-    try:
-        return struct.unpack('q', bis.read(8))[0]
-    except IOError:
-        return 0
-
-
-def object_or_list_as_string(value: Union[str, List[str]]) -> str:
-    if isinstance(value, list):
-        return f"[{','.join(value)}]"
-    return str(value)
-
-
-def map_as_string(map_: Dict[str, str]) -> str:
-    return ','.join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
-
-
-def map_with_list_as_string(map_: Dict[str, List[str]]) -> str:
-    return ','.join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
-
-
-def parse_map_string(input_str: str) -> Dict[str, str]:
-    map_ = {}
-    if input_str.startswith('{') and input_str.endswith('}'):
-        for entry in input_str[1:-1].split(','):
-            if len(entry) > 1:
-                key, value = entry.split('=')
-                map_[key] = value
-    return map_
-
-
-def csv_as_list(input_str: str) -> List[str]:
-    return [item.strip() for item in input_str.strip('[]').split(',')]
-
-
-def add_to_map(map_: Dict[str, List[str]], token: str) -> None:
-    eq_idx = token.index('=')
-    if eq_idx > 0:
-        key = token[:eq_idx]
-        map_[key] = csv_as_list(token[eq_idx+1:])
-
-
-def parse_map_list_string(input_str: str) -> Dict[str, List[str]]:
-    map_ = {}
-    if input_str.startswith('{') and input_str.endswith('}'):
-        for token in input_str[1:-1].split('],'):
-            add_to_map(map_, token)
-    return map_

amqp/router/test_route_database.py

@@ -1,24 +0,0 @@
-from unittest import TestCase
-
-
-class TestRouteDatabase(TestCase):
-    def test_get_routes(self):
-        self.fail()
-
-    def test_pattern(self):
-        self.fail()
-
-    def test_matches(self):
-        self.fail()
-
-    def test_wrap_routes(self):
-        self.fail()
-
-    def test_find_route(self):
-        self.fail()
-
-    def test_add_route(self):
-        self.fail()
-
-    def test_remove_route(self):
-        self.fail()

amqp/router/test_router_base.py

@@ -1,39 +0,0 @@
-from unittest import TestCase
-
-
-class TestRouterBase(TestCase):
-    def test_get_routing_key(self):
-        self.fail()
-
-    def test_wrap_routes(self):
-        self.fail()
-
-    def test_wrap_own_routes(self):
-        self.fail()
-
-    def test_unwrap_route_list(self):
-        self.fail()
-
-    def test_get_routes(self):
-        self.fail()
-
-    def test_find_route(self):
-        self.fail()
-
-    def test_find_route_by_message(self):
-        self.fail()
-
-    def test_add_routes(self):
-        self.fail()
-
-    def test_remove_routes(self):
-        self.fail()
-
-    def test_remove_route(self):
-        self.fail()
-
-    def test_add_route(self):
-        self.fail()
-
-    def test_add_own_route(self):
-        self.fail()

amqp/router/utils.py

@@ -1,9 +1,37 @@
+import asyncio
 import re
 
+pattern = re.compile(r"[^a-zA-Z0-9.#/\-]")
+AWAIT_SLEEP = 0.05  # seconds
+
 
 def sanitize_as_rabbitmq_name(input_str: str) -> str:
-    pattern = re.compile(r"[^a-zA-Z0-9.#/\-]")
     matcher = pattern.search(input_str)
-    token = input_str[:matcher.start()] if matcher else input_str
-    step1 = re.sub(r"[^A-Za-z0-9]", ".", token)
-    return re.sub(r"\.+", ".", step1)
+    token = input_str[: matcher.start()] if matcher else input_str
+    return sanitize_as_rabbitmq_domain_name(token)
+
+
+def sanitize_as_rabbitmq_domain_name(token: str) -> str:
+    # Step 1: Replace non-alphanumeric characters (except periods) with a period
+    step1 = re.sub(r"[^A-Za-z0-9.#]", ".", token)
+    # Step 2: Replace multiple consecutive periods with a single period
+    step2 = re.sub(r"\.+", ".", step1)
+    # Step 3: Remove trailing '.' character, if applicable
+    step_last_char = len(step2) - 1
+    return step2[:-1] if step_last_char > 0 and step2[step_last_char] == "." else step2
+
+
+async def await_future(future: asyncio.Future, sleep_time=AWAIT_SLEEP):
+    """
+    Wait for a future to complete.
+    """
+    while not future.done():
+        await asyncio.sleep(sleep_time)
+
+
+async def await_result(future: asyncio.Future, sleep_time=AWAIT_SLEEP):
+    """
+    Wait for a future to complete and return its result.
+    """
+    await await_future(future, sleep_time=sleep_time)
+    return future.result()

amqp/service/amq_message_handler.py

@@ -0,0 +1,278 @@
+import asyncio
+import os
+import threading
+import time
+from asyncio import AbstractEventLoop, Future
+from typing import Dict
+
+from aio_pika import Message
+from aio_pika.abc import AbstractIncomingMessage, AbstractRobustExchange
+from opentelemetry import trace
+from opentelemetry.trace import Tracer, TraceState
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+from amqp.adapter.backpressure_handler import BackpressureHandler
+from amqp.adapter.cleverthis_service_adapter import AMQMessage, CleverThisServiceAdapter
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.data_response_factory import DataResponseFactory
+from amqp.adapter.file_handler import FileHandler, StreamingFileHandler
+from amqp.adapter.logging_utils import logging_debug, logging_error, logging_info
+from amqp.adapter.serializer import map_as_string
+from amqp.adapter.service_message_factory import ServiceMessageFactory
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import (
+    DataMessage,
+    DataMessageMagicByte,
+    DataResponse,
+    MultipartDataMessage,
+)
+from amqp.rabbitmq.rabbit_mq_client import RabbitMQClient
+
+
+def collect_trace_info():
+    trace_map = {}
+    TraceContextTextMapPropagator().inject(
+        carrier=trace_map, context=trace.context_api.get_current()
+    )
+    return trace_map
+
+
+def get_default_trace_state():
+    return TraceState(entries={})
+
+
+def set_text(carrier, key, value):
+    carrier[key] = value
+
+
+class DataMessageHandler:
+    def __init__(
+        self,
+        service_adapter: CleverThisServiceAdapter,
+        tracer: Tracer,
+        message_factory: DataMessageFactory,
+        service_message_factory: ServiceMessageFactory,
+        reply_to_exchange: AbstractRobustExchange,
+        rabbit_mq_client: RabbitMQClient,
+        amq_configuration: AMQConfiguration,
+        loop,
+        backpressure_handler: BackpressureHandler,
+        file_handler: FileHandler = StreamingFileHandler(),
+    ):
+        self.service_adapter: CleverThisServiceAdapter = service_adapter
+        self.tracer = tracer
+        self.message_factory: DataMessageFactory = message_factory
+        self.service_message_factory: ServiceMessageFactory = service_message_factory
+        self.reply_to_exchange: AbstractRobustExchange = reply_to_exchange
+        self.rabbit_mq_client: RabbitMQClient = rabbit_mq_client
+        self.rabbit_mq_url = amq_configuration.dispatch.rabbit_mq_url
+        self.output_dir = amq_configuration.dispatch.download_dir
+        self.loop = loop
+        self.service_adapter.loop = loop
+        # Dictionary to store outstanding Futures
+        self.outstanding: Dict[str, asyncio.Future] = {}
+        self.reply_to: str | None = None
+        self.file_handler: FileHandler = file_handler
+        self.backpressure_handler: BackpressureHandler = backpressure_handler
+
+    async def inbound_data_message_callback_as_thread(self, message: AbstractIncomingMessage):
+        """
+        The callback method that is called when a DataMessage is received from RabbitMQ, starts new Thread.
+        """
+        threading.Thread(
+            target=lambda: asyncio.run(self.inbound_data_message_callback_no_thread(message))
+        ).start()
+
+    async def inbound_data_message_callback_no_thread(self, message: AbstractIncomingMessage):
+        """
+        Receives the DataMessage as bytes from RabbitMQ queue and deserializes it to the DataMessage object
+        Ensures Jaeger trace-id propagation.
+        Invokes custom handler/mapper/adapter method that corresponds to this message
+        :param message:
+        :return:
+        """
+        start = time.time()
+        amq_message = await self.reconstitute_data_message(message)
+        if amq_message is not None:
+            self.ensure_trace_info(amq_message)
+            if amq_message.magic() == DataMessageMagicByte.HTTP_REQUEST.value:
+                await self.run_http_request_magic(message, amq_message, self.loop)
+            else:
+                logging_error(
+                    "Unknown MAGIC value: [%s], don't know how to interpret the message",
+                    amq_message.magic(),
+                )
+                asyncio.run_coroutine_threadsafe(message.nack(requeue=False), self.loop)
+                return
+        else:
+            logging_error("######[%s] No valid AMQ Message present.", message.delivery_tag)
+        self.record_duration(start, message.delivery_tag)
+        asyncio.run_coroutine_threadsafe(message.ack(multiple=False), self.loop)
+
+    async def run_http_request_magic(
+        self,
+        message: AbstractIncomingMessage,
+        amq_message: DataMessage,
+        loop: AbstractEventLoop,
+    ) -> None:
+        """
+        Synchronous version of on_message method. This is a wrapper around the async on_message method.
+        It also counts parallel executions and handles backpressure.
+        :param message: RabbitMQ raw message
+        :param amq_message: DataMessage
+        :param loop: Event loop
+        :return: DataResponse
+        """
+        # Increment the counter for parallel executions
+        self.backpressure_handler.increase_parallel_executions()
+        logging_info(
+            f"PARALLEL: Current Capacity [{self.backpressure_handler.current_parallel_executions}] of [{self.backpressure_handler.parallel_workers}]"
+        )
+        if message.reply_to:
+            self.reply_to = message.reply_to
+
+        try:
+            await self.backpressure_handler.check_overload_condition()
+            _a_message = AMQMessage(amq_message, self.rabbit_mq_client.connection)
+            _response: DataResponse = await self.service_adapter.on_message(_a_message)
+            logging_debug(f"Result in thread: {_response}")
+            try:
+                await self.send_reply(message.reply_to, _response)
+            except Exception as e:
+                logging_error(f"Processing message: {e}")
+                asyncio.run_coroutine_threadsafe(message.nack(requeue=False), loop=loop)
+
+            if isinstance(amq_message, MultipartDataMessage):
+                # remove the downloaded files from the output_dir
+                for file_id, filename in amq_message.extra_data.items():
+                    try:
+                        logging_info(
+                            f"[{message.delivery_tag}] Deleting file: {file_id} in {filename}"
+                        )
+                        os.remove(filename)
+                    except OSError as e:
+                        logging_error(f"Deleting file {filename}: {e}")
+        except Exception as e:
+            logging_error(f"Request handler: {e}")
+
+        self.backpressure_handler.decrease_parallel_executions()
+
+    async def reconstitute_data_message(
+        self, message: AbstractIncomingMessage
+    ) -> DataMessage | None:
+        amq_message: DataMessage | None = None
+        delivery_tag = message.delivery_tag
+        addressing: int = message.headers.get("clevermicro.addressing", 0)
+        if addressing == 0:
+            amq_message = DataMessageFactory.from_bytes(message.body)
+        else:
+            amq_message = await DataMessageFactory.from_stream(
+                message.body, rabbitmq_url=self.rabbit_mq_url, loop=self.loop
+            )
+            logging_info(f"######[{delivery_tag}] AMQ Message from stream: {amq_message}")
+            if amq_message is not None:
+                extra_data = await self.file_handler.on_message(
+                    message=message,
+                    json_data=amq_message.body(),
+                    rabbitmq_url=self.rabbit_mq_url,
+                    loop=self.loop,
+                    output_dir=self.output_dir,
+                )
+                logging_info(f"######[{delivery_tag}] Multipart download result: {extra_data}")
+                amq_message = MultipartDataMessage(amq_message, extra_data)
+        if amq_message:
+            logging_info(
+                "######[%s] AMQ Message received, ctag=%s, msgId=%s, %s:%s, traceInfo=%s",
+                message.delivery_tag,
+                message.consumer_tag,
+                amq_message.id(),
+                amq_message.method(),
+                amq_message.path(),
+                map_as_string(amq_message.trace_info()),
+            )
+        return amq_message
+
+    def ensure_trace_info(self, amq_message: DataMessage):
+        propagator = TraceContextTextMapPropagator()
+        context = propagator.extract(
+            carrier=amq_message.trace_info(), context=trace.context_api.get_current()
+        )
+        self.tracer.start_span(name="AMQ-Adapter-Consumer", context=context)
+        # logging_info("CTX=%s", context)
+        # current_span = self.tracer.start_span("AMQ-Adapter.Processing.Request", parent=parent_span)
+        # context_with_span = trace.set_span_in_context(current_span, context)
+        # logging.info(" [*] CTX2=%s", context_with_span)
+        # propagator.inject(context_with_span, amq_message.trace_info, self.set_text)
+        # logging.info(" [***] NEW TI=%s CTX=%s SCOPE=%s",
+        #             map_as_string(amq_message.trace_info()), context_with_span, context)
+
+    def record_duration(self, start, delivery_tag):
+        global last_data_message_time
+        last_data_message_time = time.time()
+        logging_info(
+            "######[%s] Done, single ACK: %sms.",
+            delivery_tag,
+            last_data_message_time - start,
+        )
+
+    async def send_reply(self, reply_to: str, response: DataResponse):
+        """
+        The Service side code. When the CleverXXX service has output ready, in DataResponse object,
+        call this method to return the reply back to the caller
+        :param reply_to:connection.channel
+        :param response:
+        :return:
+        """
+        if reply_to:
+            pika_message: Message = Message(
+                body=DataResponseFactory.serialize(response),
+                correlation_id=response.id(),
+                content_type=(
+                    response.content_type()[0]
+                    if isinstance(response.content_type(), list)
+                    else response.content_type()
+                ),
+            )
+            _res = asyncio.run_coroutine_threadsafe(
+                self.reply_to_exchange.publish(message=pika_message, routing_key=reply_to),
+                self.loop,
+            )
+            logging_debug(
+                "###### DONE Reply Published to exchg:(%s) To=%s, res=%s",
+                self.reply_to_exchange.name,
+                reply_to,
+                _res,
+            )
+
+    async def reply_received_callback(self, message: AbstractIncomingMessage):
+        """
+        The reply for DataMessage that we sent out earlier - must match the received response with
+        the original request and respond to the caller.
+        :param message:
+        :return:
+        """
+        reply_id = message.correlation_id
+        delivery_tag = message.delivery_tag
+        debug = f"######[{delivery_tag}] (RPC REPLY) env: {message.headers}, props: {message.properties}, BODY: {message.body.decode()}"
+        logging_debug(debug)
+        # TODO - figure out implementation of this part, as this is sending reply back either to a service,
+        #        or to the user
+        logging_info(f"OUTSTANDING RECV: {self.outstanding}, size={len(self.outstanding)}")
+
+        future: Future = self.outstanding.pop(reply_id, None)
+        # if reply is None:
+        # logging.warn(" [W] ######[%s] IGNORING REPLY. No outstanding request for ID %s",
+        #             delivery_tag, reply_id)
+        # else:
+        # result = DataResponseFactory.from_bytes(body)
+        # reply.tracing_span().end()
+        # TODO - figure out the right response format (not needed for Clever Swarm v0.1)
+        # that currently runs Java version, this method here is for the future compatibility to enable
+        # internal, service-2-service communication, and the proper response type will be decided than
+        # reply.response().set_result(Response.ok(result.body, content_type).build())
+        if future:
+            response: DataResponse = DataResponseFactory.from_bytes(message.body)
+            # Complete the Future with the response data
+            future.set_result(response.body())
+
+        await message.ack(multiple=False)

amqp/service/amq_service.py

@@ -1,84 +1,129 @@
-import os
+import asyncio
 import logging
-from functools import partial
-from opentelemetry import trace
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import (
-    ConsoleSpanExporter,
-    SimpleSpanProcessor
-)
-from opentelemetry.propagate import set_global_textmap
-from opentelemetry.propagators.tracecontext import TraceContextTextMapPropagator
+import logging.config
+import os
+from asyncio import Future
+from threading import Thread
+from typing import Optional
+
+from aio_pika.abc import AbstractRobustExchange
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
 from amqp.adapter.amq_route_factory import AMQRouteFactory
-from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
-from amqp.rabbitmq.rabbit_mq_consumer import RabbitMQConsumer
-from amqp.service.service_message_handler import AMQServiceMessageHandler
+from amqp.adapter.backpressure_handler import BackpressureHandler
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.logging_utils import logging_info, logging_warning
+from amqp.adapter.service_message_factory import ServiceMessageFactory
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import DataMessage
+from amqp.rabbitmq.rabbit_mq_client import RabbitMQClient
+from amqp.service.amq_message_handler import DataMessageHandler
+from amqp.service.tracing import TracingConfig, initialize_telemetry
+
+logging.basicConfig(level=logging.DEBUG)
+logging.info(f"CWD = {os.getcwd()}")
+logging_conf_path = os.path.join(os.getcwd(), "amqp", "service", "logging.conf")
+if os.path.exists(logging_conf_path):
+    logging.config.fileConfig(logging_conf_path)
+else:
+    logging_conf_path = os.path.join(os.getcwd(), "logging.conf")
+    if os.path.exists(logging_conf_path):
+        logging.config.fileConfig(logging_conf_path)
 
 
 class AMQService:
     CLEVER_AMQP_CLIENT_VERSION = os.getenv("CLEVER_AMQP_CLIENT_VERSION")
     MAX_WAIT = 300  # defaults to 5 minutes (300 seconds)
 
-    def __init__(self, amq_configuration, http_adapter):
-        self.amq_configuration = amq_configuration
-        self.http_adapter = http_adapter
-        self.message_factory = AMQMessageFactory.get_instance(self.amq_configuration.amq_adapter().generator_id())
-        self.rabbit_mq_consumer = RabbitMQConsumer(self.amq_configuration)
-        self.amq_service_message_handler = AMQServiceMessageHandler(self.http_adapter, self.get_tracer())
-
-    def init(self):
-        CleverMicroServiceMessageFactory.process_name(self.amq_configuration.amq_adapter().service_name())
-        logging.info("*********  AMQService.isRouter = %s ***********", self.amq_configuration.amq_adapter().is_router())
+    def __init__(self, amq_configuration: AMQConfiguration, service_adapter=None):
         logging.info("***********************************************")
         logging.info("**********  AMQ   Service   *******************")
         logging.info("***********************************************")
+        self.loop = asyncio.new_event_loop()
 
-        self.setup_opentelemetry()
-        self.register_routes()
-        self.register_rpc_handler()
-        self.rabbit_mq_consumer.request_routes()
+        self.amq_configuration = amq_configuration
+        self.service_adapter = service_adapter
+        self.data_message_factory = DataMessageFactory.get_instance(
+            self.amq_configuration.amq_adapter.generator_id
+        )
+        self.rabbit_mq_client = RabbitMQClient(self.amq_configuration)
+        self.tracer = initialize_telemetry(
+            tracing_config=TracingConfig(
+                service_name=amq_configuration.amq_adapter.service_name,
+                service_version=os.getenv("SERVICE_VERSION", "unknown"),
+            )
+        )
+        self.amq_data_message_handler = None
+        self.service_message_factory = ServiceMessageFactory(
+            self.amq_configuration.amq_adapter.service_name
+        )
+        self.backpressure_thread: Optional[Thread] = None
+
+    def run(self):
+        # self.init(loop) will initialize RabbitMQ connection
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_until_complete(self.init(self.loop, self.service_adapter))
+
+        logging.info("***********************************************")
+        logging.info("******  AMQ   Service  Init  Complete *********")
+        logging.info("***********************************************")
+        self.loop.run_forever()
+
+    async def init(self, loop, service_adapter):
+        _reply_to_exchange: AbstractRobustExchange = await self.rabbit_mq_client.init(loop)
+        backpressure_handler = BackpressureHandler(
+            channel=self.rabbit_mq_client.channel,
+            loop=loop,
+            config=self.amq_configuration,
+        )
+        self.amq_data_message_handler = DataMessageHandler(
+            service_adapter,
+            self.tracer,
+            self.data_message_factory,
+            self.service_message_factory,
+            _reply_to_exchange,
+            self.rabbit_mq_client,
+            self.amq_configuration,
+            loop=loop,
+            backpressure_handler=backpressure_handler,
+        )
+        await self.register_routes()
+        await self.rabbit_mq_client.request_routes()
+        self.backpressure_thread = backpressure_handler.start_backpressure_monitor()
 
     def cleanup(self):
-        logging.info(" [DBG] AMQService.cleanup() rmqc=%s", self.rabbit_mq_consumer)
-        if self.rabbit_mq_consumer:
-            self.rabbit_mq_consumer.cleanup()
+        logging_info("AMQService.cleanup() rmqc=%s", self.rabbit_mq_client)
+        if self.rabbit_mq_client:
+            self.rabbit_mq_client.cleanup()
 
-    def reinitialize_if_disconnected(self):
-        if not self.rabbit_mq_consumer.channel or not self.rabbit_mq_consumer.channel.is_open:
-            self.init()
+    async def reinitialize_if_disconnected(self):
+        if not self.rabbit_mq_client.channel or self.rabbit_mq_client.channel.is_closed:
+            self.rabbit_mq_client = RabbitMQClient(self.amq_configuration)
+            await self.rabbit_mq_client.init(loop=self.loop)
+            await self.register_routes()
+            await self.rabbit_mq_client.request_routes()
 
-    def setup_opentelemetry(self):
-        trace.set_tracer_provider(TracerProvider())
-        trace.get_tracer_provider().add_span_processor(
-            SimpleSpanProcessor(ConsoleSpanExporter())
-        )
-        set_global_textmap(TraceContextTextMapPropagator())
-        self.tracer = trace.get_tracer("com.cleverthis")
-        logging.info("OTEL %s %s %s", trace.get_logger_bridge(), trace.get_meter_provider(), trace.get_tracer_provider())
-        logging.info("OTEL TRACER %s", trace.get_propagators())
-
-    def register_routes(self):
-        route_mapping = self.amq_configuration.amq_adapter().route_mapping()
+    async def register_routes(self) -> AbstractRobustExchange | None:
+        route_mapping = self.amq_configuration.amq_adapter.route_mapping
         if AMQRouteFactory.validate(route_mapping):
             try:
-                self.rabbit_mq_consumer.register_routes()
-                self.rabbit_mq_consumer.register_deliver_callback(
-                    partial(self.amq_service_message_handler.on_delivery_callback_http, self.rabbit_mq_consumer.channel)
+                await self.rabbit_mq_client.register_inbound_routes(
+                    route_mapping,
+                    self.amq_data_message_handler.inbound_data_message_callback_no_thread,
                 )
-                self.rabbit_mq_consumer.register_rpc_handler(
-                    self.amq_service_message_handler.on_reply_consumer(self.rabbit_mq_consumer.channel)
+                return await self.rabbit_mq_client.register_data_reply_callback(
+                    self.amq_data_message_handler.reply_received_callback
                 )
             except Exception as e:
                 raise RuntimeError(e)
         elif route_mapping:
-            logging.info(" [W] route [%s] failed validation, no routes were registered.", route_mapping)
-
-    def register_rpc_handler(self):
-        try:
-            self.rabbit_mq_consumer.register_rpc_handler(
-                self.amq_service_message_handler.on_reply_consumer(self.rabbit_mq_consumer.get_channel())
+            logging_warning(
+                "route [%s] failed validation, no routes were registered.",
+                route_mapping,
             )
-        except Exception as err:
-            raise
+        return None
+
+    def send_message(self, message: DataMessage, future: Future):
+        self.amq_data_message_handler.outstanding[str(message.id())] = future
+
+    def remove_outstanding_future(self, message_id: str):
+        self.amq_data_message_handler.outstanding.pop(message_id, None)

amqp/service/logging.conf

@@ -0,0 +1,34 @@
+[loggers]
+keys=root,amqp_adapter
+
+[handlers]
+keys=consoleHandler,fileHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_amqp_adapter]
+level=DEBUG
+handlers=consoleHandler,fileHandler
+qualname=amqp_adapter
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=FileHandler
+level=DEBUG
+formatter=simpleFormatter
+args=('AMQ_PyAdapter.log',)
+
+[formatter_simpleFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+datefmt=%Y-%m-%d %H:%M:%S

amqp/service/service_message_handler.py

@@ -1,143 +0,0 @@
-import logging
-import time
-
-from typing import Callable, Dict
-from concurrent.futures import Future
-from opentelemetry import trace
-from opentelemetry.propagate import get_global_textmap, set_global_textmap
-from opentelemetry.propagators.tracecontext import TraceContextTextMapPropagator
-from opentelemetry.trace import Span, Tracer, TraceState
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-from amqp.model.model import AMQResponse
-from amqp.router.serializer import map_as_string
-
-
-class AMQServiceMessageHandler:
-    def __init__(self, http_adapter, tracer):
-        self.http_adapter = http_adapter
-        self.tracer = tracer
-        self.outstanding = {}
-
-    def on_delivery_callback_http(self, channel, consumerTag, delivery):
-        start = time.time()
-        deliveryTag = delivery.envelope.delivery_tag
-        debug = f" [DBG] ######[{deliveryTag}] (AMQ MESSAGE), env: {delivery.envelope}, props: {delivery.properties}"
-        logging.info(debug)
-
-        amq_message = AMQMessageFactory.from_bytes(delivery.body)
-        logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, traceInfo=%s",
-                    deliveryTag, consumerTag, amq_message.id, map_as_string(amq_message.trace_info))
-
-        propagator = TraceContextTextMapPropagator.get_instance()
-        context = propagator.extract(trace.get_current_context(), amq_message.trace_info    , self.get_text_getter())
-        parent_span = Span.from_context(context)
-        context = context.with_span(parent_span)
-
-        with context.activate():
-            logging.info(" [*] CTX=%s", context)
-            current_span = self.tracer.start_span("AMQ-Adapter.Processing.Request", parent=parent_span)
-            context_with_span = trace.set_span_in_context(current_span, context)
-            logging.info(" [*] CTX2=%s", context_with_span)
-            propagator.inject(context_with_span, amq_message.trace_info, self.set_text)
-            logging.info(" [***] NEW TI=%s CTX=%s SCOPE=%s",
-                        map_as_string(amq_message.trace_info), context_with_span, context)
-
-            if amq_message.magic == AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST:
-                future = self.http_adapter.handle(amq_message)
-                future.add_done_callback(
-                    lambda f: self.on_http_response(delivery, deliveryTag, channel, f.result())
-                )
-            else:
-                logging.error(" [E] Unknown MAGIC value: [%s], don't know how to interpret the message",
-                              amq_message.magic())
-
-            duration = time.time() - start
-            if "___NACK___" in str(amq_message.body()):
-                logging.info(" [x] ######[%s] Done, single NACK: %sms.", deliveryTag, duration)
-                channel.basic_nack(deliveryTag, False, False)
-            else:
-                logging.info(" [x] ######[%s] Done, single ACK: %sms.", deliveryTag, duration)
-                channel.basic_ack(deliveryTag, False)
-
-    def on_http_response(self, delivery, deliveryTag, channel, http_response):
-        response = AMQResponse(
-            delivery.properties.correlation_id,
-            http_response.status_code,
-            http_response.headers.get("Content-Type"),
-            http_response.body,
-            http_response.status_message,
-            "",
-            self.collect_trace_info(trace.get_current_context())
-        )
-        try:
-            self.send_reply(delivery.properties.reply_to, deliveryTag, channel, response)
-        except Exception as e:
-            raise RuntimeError(e)
-
-
-    class AMQServiceMessageHandler:
-        def __init__(self, http_adapter, tracer):
-            self.http_adapter = http_adapter
-            self.tracer = tracer
-            self.outstanding: Dict[str, 'ResponseHolder'] = {}
-
-        def collect_trace_info(self, context):
-            map = {}
-            TraceContextTextMapPropagator.get_instance().inject(
-                context if context else trace.get_current_context(),
-                map, self.set_text
-            )
-            return map
-
-        def send_reply(self, reply_to, delivery_tag, channel, response):
-            if reply_to:
-                logging.info(" [*] ######[%s] AMQ Message Reply, To=%s, msgId=%s",
-                             delivery_tag, reply_to, response.id())
-                reply_props = pika.BasicProperties(
-                    correlation_id=response.id(),
-                    content_type=response.content_type()
-                )
-                channel.basic_publish(
-                    RouterBase.AMQ_REPLY_TO_EXCHANGE,
-                    reply_to,
-                    reply_props,
-                    AMQResponseFactory.serialize(response)
-                )
-
-        def on_reply_consumer(self, channel):
-            def handle_delivery(consumerTag, envelope, properties, body):
-                reply_id = properties.correlation_id
-                content_type = properties.content_type
-                delivery_tag = envelope.delivery_tag
-                debug = f" [DBG] ######[{delivery_tag}]     (RPC REPLY) env: {envelope}, props: {properties}, BODY: {body.decode()}"
-                logging.info(debug)
-                print(f"OUTSTANDING RECV: {self.outstanding}, size={len(self.outstanding)}")
-
-                with self.outstanding.lock:
-                    reply = self.outstanding.pop(reply_id, None)
-                    if reply is None:
-                        logging.warn(" [W] ######[%s] IGNORING REPLY. No outstanding request for ID %s",
-                                     delivery_tag, reply_id)
-                    else:
-                        result = AMQResponseFactory.from_bytes(body)
-                        reply.tracing_span().end()
-                        reply.response().set_result(Response.ok(result.body(), content_type).build())
-
-                channel.basic_ack(delivery_tag, False)
-
-            return handle_delivery
-
-        def register_response_holder(self, id, response_holder):
-            self.outstanding[id] = response_holder
-            logging.info(" [*] AMQMessage Sent, OUTSTANDING response holder registered: %s, size=%s",
-                         self.outstanding, len(self.outstanding))
-
-        def get_default_trace_state(self):
-            return TraceState(entries={})
-
-        def get_text_getter(self):
-            return lambda map, key: map.get(key)
-
-        def set_text(self, carrier, key, value):
-            carrier[key] = value

amqp/service/tracing.py

@@ -1,89 +1,196 @@
-from opentelemetry import metrics, trace
-from opentelemetry.instrumentation.flask import FlaskInstrumentor
-from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+"""
+OpenTelemetry configuration for tracing and metrics.
 
-from prometheus_client import start_http_server
-from opentelemetry.exporter.prometheus import PrometheusMetricReader
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+This module provides functionality to initialize and configure OpenTelemetry
+for both tracing (Jaeger via OTLP) and metrics (Prometheus) in the AMQ adapter.
+"""
 
+import logging
 import os
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
 
-def initialize_trace(flask_app, name):
-    # Resource can be required for some backends, e.g. Jaeger or Prometheus
-    # If resource wouldn't be set - traces wouldn't appear in Jaeger.
-    # The 'name' value is the name shown in Jaeger.
-    resource = Resource(attributes={
-        SERVICE_NAME: name if name else os.environ.get('APPLICATION_NAME', 'Python App')
-    })
-    #
-    # 1. Tracing (Jaeger) configuration
-    # ----------------------------------
-    #
-    # Initialize the tracer provider. This is abstract API call.
-    # The actual implementation is given by the included dependency
-    # The opentelemetry default is the OTLP trace provider.
-    trace.set_tracer_provider(TracerProvider(resource=resource))
-    #
-    # Configure the OTLP exporter (or HTTP exporter the same way)
-    # the recommended configuration is via environment variables. The mandatory ones:
-    #       OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
-    #       OTEL_EXPORTER_OTLP_INSECURE: "true"
-    # for more see here: https://opentelemetry-python.readthedocs.io/en/latest/exporter/otlp/otlp.html
-    # alternatively these values can be provided as input values to OTLPSpanExporter() constructor.
-    #
-    otlp_exporter = OTLPSpanExporter()
-    #
-    # Add the exporters to the batch span processor
-    span_processor = BatchSpanProcessor(otlp_exporter)
-    trace.get_tracer_provider().add_span_processor(span_processor)
+from opentelemetry import metrics, trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.exporter.prometheus import PrometheusMetricReader
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import SERVICE_NAME, SERVICE_VERSION, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+from opentelemetry.trace import Tracer
+from prometheus_client import start_http_server
 
-    print(' * Instrumenting Flask application..... ')
-    FlaskInstrumentor().instrument_app(flask_app)
-    #
-    # Python Flask is now automatically instrumented to export details of each REST endpoint call.
-    # For Django based app, use 'opentelemetry-instrumentation-django' import instead. see here:
-    # https://opentelemetry-python.readthedocs.io/en/latest/examples/django/README.html for example
-    #
-    # If the used library has no instrumentation support in opentelemetry.io, then the trace
-    # can be started manually. In this code,
-    # 'trace' should be global shared variable, initialized once per application (see tracing.py)
-    # 'tracer' can be shared global or local variable, application can use multiple uniquely
-    #          named tracers, Jaeger will group the generated traces by the name of the tracer
-    #          that was used to start the Span. See this example code:
-    #
-    # tracer = trace.get_tracer(name)
-    #
-    # @app.route("/server_request")
-    # def server_request():
-    #     with tracer.start_as_current_span(
-    #         "server_request",
-    #         context=extract(request.headers),
-    #         kind=trace.SpanKind.SERVER,
-    #         attributes=collect_request_attributes(request.environ),
-    #     ):
-    #         # do the work here, like print something and return response
-    #         print(request.args.get("param"))
-    #         return "served"
-    #
-    # Note:
-    #    the otlp code looks for HTTP header named 'traceparent' that, if present, should contain
-    #    the otlp parent span identification, and the span created here will be made a child
-    #    of that parent span, indicating visually the progression of requests via different services
-    #    for as long as the current span is provided in the traceparent header to next service.
+logger = logging.getLogger(__name__)
 
 
-    # 2. Metrics (Prometheus) configuration
-    # -------------------------------------
-    #
-    # Start Prometheus client
-    start_http_server(port=9464, addr="localhost")
-    # Initialize PrometheusMetricReader which pulls metrics from the SDK
-    # on-demand to respond to scrape requests (unlike Jaeger, which worls with push model - application
-    # pushes traces when those become available to Jaeger's collector. Prometheus works with pull model
-    # where it periodically polls pre-configured and/or discovered endpoints for metrics.
-    reader = PrometheusMetricReader()
-    provider = MeterProvider(resource=resource, metric_readers=[reader])
-    metrics.set_meter_provider(provider)
+@dataclass
+class TracingConfig:
+    """Configuration for OpenTelemetry tracing."""
+
+    service_name: str
+    service_version: str
+    otlp_endpoint: str = "http://localhost:4317"  # Default Jaeger OTLP endpoint
+    insecure: bool = True
+    debug: bool = False
+    additional_attributes: Optional[Dict[str, Any]] = None
+
+
+@dataclass
+class MetricsConfig:
+    """Configuration for OpenTelemetry metrics."""
+
+    prometheus_port: int = 9464
+    prometheus_host: str = "localhost"
+    export_interval_millis: int = 30000
+    additional_attributes: Optional[Dict[str, Any]] = None
+
+
+def create_resource(config: TracingConfig) -> Resource:
+    """Create an OpenTelemetry resource with service information."""
+    attributes = {
+        SERVICE_NAME: config.service_name,
+        SERVICE_VERSION: config.service_version,
+    }
+
+    if config.additional_attributes:
+        attributes.update(config.additional_attributes)
+
+    return Resource.create(attributes)
+
+
+def setup_otlp_exporter(config: TracingConfig) -> OTLPSpanExporter:
+    """Configure and create an OTLP exporter for Jaeger."""
+    return OTLPSpanExporter(endpoint=config.otlp_endpoint, insecure=config.insecure)
+
+
+def setup_trace_provider(config: TracingConfig, resource: Resource) -> TracerProvider:
+    """Set up the TracerProvider with OTLP exporter."""
+    provider = TracerProvider(resource=resource)
+
+    # Add OTLP exporter for Jaeger
+    otlp_exporter = setup_otlp_exporter(config)
+    provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
+
+    # Add console exporter in debug mode
+    if config.debug:
+        provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+
+    return provider
+
+
+def setup_metrics_provider(config: MetricsConfig, resource: Resource) -> MeterProvider:
+    """Set up the MeterProvider with Prometheus export."""
+    try:
+        # Start Prometheus HTTP server
+        start_http_server(port=config.prometheus_port, addr=config.prometheus_host)
+        logger.info(
+            f"Started Prometheus metrics server on {config.prometheus_host}:{config.prometheus_port}"
+        )
+
+        # Create Prometheus reader
+        prometheus_reader = PrometheusMetricReader()
+
+        # Create periodic reader for custom metrics
+        periodic_reader = PeriodicExportingMetricReader(
+            prometheus_reader, export_interval_millis=config.export_interval_millis
+        )
+
+        # Create and configure meter provider
+        return MeterProvider(resource=resource, metric_readers=[prometheus_reader, periodic_reader])
+    except Exception as e:
+        logger.error(f"Failed to set up metrics provider: {str(e)}")
+        # Return a no-op meter provider
+        return MeterProvider(resource=resource)
+
+
+def initialize_telemetry(
+    tracing_config: Optional[TracingConfig] = None,
+    metrics_config: Optional[MetricsConfig] = None,
+) -> Tracer:
+    """
+    Initialize OpenTelemetry with tracing and metrics.
+
+    Args:
+        tracing_config: Configuration for tracing
+        metrics_config: Configuration for metrics
+
+    Returns:
+        Tracer: Configured OpenTelemetry tracer
+
+    Example:
+        ```python
+        config = TracingConfig(
+            service_name="amq-adapter",
+            service_version="1.0.0",
+            otlp_endpoint="http://jaeger:4317",
+            debug=True
+        )
+
+        metrics_config = MetricsConfig(
+            prometheus_port=9464,
+            prometheus_host="0.0.0.0"
+        )
+
+        tracer = initialize_telemetry(config, metrics_config)
+        ```
+    """
+    # Use default configs if none provided
+    if tracing_config is None:
+        tracing_config = TracingConfig(
+            service_name=os.getenv("SERVICE_NAME", "amq-adapter"),
+            service_version=os.getenv("SERVICE_VERSION", "unknown"),
+            otlp_endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317"),
+            insecure=os.getenv("OTEL_INSECURE_MODE", "true").lower() == "true",
+            debug=os.getenv("OTEL_DEBUG", "false").lower() == "true",
+        )
+
+    if metrics_config is None:
+        metrics_config = MetricsConfig(
+            prometheus_port=int(os.getenv("PROMETHEUS_PORT", "9464")),
+            prometheus_host=os.getenv("PROMETHEUS_HOST", "localhost"),
+            export_interval_millis=int(os.getenv("METRICS_EXPORT_INTERVAL", "30000")),
+        )
+
+    try:
+        # Create resource
+        resource = create_resource(tracing_config)
+
+        # Set up tracing
+        provider = setup_trace_provider(tracing_config, resource)
+        trace.set_tracer_provider(provider)
+        logger.info(f"Initialized tracing for service {tracing_config.service_name}")
+
+        # Set up metrics
+        metrics_provider = setup_metrics_provider(metrics_config, resource)
+        metrics.set_meter_provider(metrics_provider)
+        logger.info("Initialized metrics export")
+
+        # Return configured tracer
+        return trace.get_tracer(
+            tracing_config.service_name,
+            tracing_config.service_version,
+        )
+
+    except Exception as e:
+        logger.error(f"Failed to initialize telemetry: {str(e)}")
+        # Return a no-op tracer in case of failure
+        return trace.get_tracer("no-op")
+
+
+def create_span(tracer: Tracer, name: str, attributes: Optional[Dict[str, Any]] = None):
+    """
+    Create a new span with the given name and attributes.
+
+    Args:
+        tracer: The OpenTelemetry tracer
+        name: Name of the span
+        attributes: Optional attributes to add to the span
+
+    Returns:
+        A context manager that creates and manages a span
+    """
+    return tracer.start_as_current_span(
+        name,
+        attributes=attributes or {},
+    )

cleverbrag/clever_brag_adapter.py

@@ -0,0 +1,125 @@
+"""
+This module provides the CleverBRAG AMQP Adapter for handling API requests via AMQP.
+The code IS MEANT to be part of CleverBRAG codebase, and has direct dependencies on the CleverBRAG codebase.
+It is NOT MEANT to be used as a standalone module. It WILL NOT run outside the CleverBRAG context.
+"""
+
+from threading import Thread
+from typing import Any, List
+from uuid import UUID
+
+# CleverBrag specific imports
+import core
+from fastapi.routing import APIRoute
+from fastapi.security.http import HTTPAuthorizationCredentials
+from pydantic import BaseModel
+from shared.abstractions import R2RSerializable, User
+
+# AMQP specific imports
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.service.amq_service import AMQService
+
+
+class BRAGArgument:
+    """
+    This class is used to represent a BRAG API input argument. It is uses FastAPI annotation to define the arguments
+    (the argument type) that are passed as input arguments to the BRAG endpoint.
+    """
+
+    def __init__(self, arg: Any):
+        self.arg = arg
+        self.module = arg.__module__
+        self.cls = arg.__name__ if hasattr(arg, "__name__") else str(arg)
+        self.is_r2r_serializable = issubclass(arg, R2RSerializable)
+        self.is_builtin = self.module == "builtins"
+        self.is_uuid = self.cls == "UUID"
+        self.is_optional = self.cls == "Optional"
+        self.is_list = self.cls == "list"
+        self.is_dict = self.cls == "dict"
+        self.is_base_model = issubclass(arg, BaseModel)
+        self.is_nested = self.is_list or self.is_optional or self.is_dict
+        if self.is_nested and hasattr(arg, "__args__"):
+            self.nested = BRAGArgument(getattr(arg, "__args__")[0])
+            if self.is_dict:
+                self.nested = (self.nested, BRAGArgument(getattr(arg, "__args__")[1]))
+        else:
+            self.nested = None
+            self.is_nested = False
+
+    def __repr__(self):
+        return f"{self.module}.{self.cls}" + (f"[{self.nested}]" if self.is_nested else "")
+
+
+def convert_to_argument(arg: BRAGArgument, value: str) -> Any:
+    """
+    Converts the argument to a string representation.
+    """
+    try:
+        if arg.cls == "str":
+            return value
+        elif arg.cls == "bool":
+            return str(value).lower() == "true"
+        elif arg.cls == "float":
+            return float(value)
+        elif arg.cls == "int":
+            return int(value)
+        elif arg.is_uuid:
+            return UUID(value)
+        elif arg.is_optional:
+            return convert_to_argument(arg.nested, value) if arg.nested else None
+        elif arg.is_list:
+            return (
+                [convert_to_argument(arg.nested, v.strip()) for v in value.split(",")]
+                if arg.nested
+                else value.split()
+            )
+        elif arg.is_dict:
+            return (
+                {
+                    convert_to_argument(arg.nested[0], k.strip()): convert_to_argument(
+                        arg.nested[1], v.strip()
+                    )
+                    for k, v in (item.split(":") for item in value.split(","))
+                }
+                if arg.nested
+                else dict(item.split(":") for item in value.split(","))
+            )
+        elif arg.is_base_model:
+            return arg.arg.parse_raw(value)
+        else:
+            print(f"Unsupported argument type: {type(arg)}")
+    except Exception as e:
+        print(f"Error converting argument {arg}: {e}")
+    return None
+
+
+class CleverBragAmqpAdapter(CleverThisServiceAdapter):
+    """
+    CleverBRAG AMQP Adapter for CleverBRAG API. Implements the interface to CleverBRAG API.
+    """
+
+    def __init__(
+        self,
+        amq_configuration: AMQConfiguration,
+        routes: List[APIRoute],
+        auth_provider: core.base.providers.auth.AuthProvider,
+        extra_init_data: dict,
+    ):
+        super().__init__(
+            amq_configuration, routes, None, "auth_user", extra_init_data=extra_init_data
+        )
+        self.auth_provider = auth_provider
+        _amq_service: AMQService = AMQService(amq_configuration, self)
+        self.thread = Thread(target=_amq_service.run)
+        self.thread.start()
+
+    async def get_current_user(self, token: str) -> User:
+        """
+        Overrides the default get_current_user method to use the token from the AMQP message.
+        :param token: JWT token from the AMQP message
+        :return: User object
+        """
+        return await self.auth_provider.auth_wrapper(
+            HTTPAuthorizationCredentials(scheme="Bearer", credentials=token)
+        )

cleverbrag/main.py

@@ -0,0 +1,44 @@
+from clever_brag_adapter import BRAGArgument, CleverBragAmqpAdapter
+
+from amqp.config.amq_configuration import AMQConfiguration
+
+
+class R2RApp:
+    pass
+
+
+def create_r2r_app():
+    return R2RApp()
+
+
+if __name__ == "__main__":
+
+    #
+    # r2rapp: R2RApp = await create_r2r_app()
+    #
+    # In the standalone test context, need to start the loop as well. In the actual BRAG context,
+    # use 'await' instead.
+    r2rapp: R2RApp = create_r2r_app()
+
+    adapter: CleverBragAmqpAdapter = CleverBragAmqpAdapter(
+        AMQConfiguration("./application.properties.local"),
+        r2rapp.app.routes,
+        r2rapp.providers.auth,
+        {"app": r2rapp.config.app},
+    )
+
+    for method, routes in adapter.routes.items():
+        print(f"Method: {method}")
+        for _route in routes:
+            print(
+                f"  Route: {_route.path} -> {_route.endpoint.__module__}.{_route.endpoint.__name__}"
+            )
+            _args = getattr(_route.endpoint, "__annotations__", {})
+            _verified_args = []
+            for arg in _args:
+                _arg = BRAGArgument(_args[arg])
+                _verified_args.append(_arg)
+            print(f"    Arguments: {', '.join(map(str, _verified_args))}")
+            print("---------------------------")
+    print("Press ^C to exit...")
+    adapter.thread.join()

cleverbrag/tests/test_collections_router.sh

@@ -0,0 +1,149 @@
+echo "#!/bin/bash"
+
+. ./test_shared_functions.sh
+
+extract_document_id() {
+  R=$1
+  C=$(echo $R | sed 's/[0-9][0-9][0-9]$//')
+  #echo ">>${C}<<"
+  if [[ ! -z "$C" ]] && [[ "$C" == *results* ]]; then
+    COLLECTION_IDS=($(echo $C | jq -r '.results[] | select(.document_count >= 1) | .id' | tr '\n' ' '))
+    echo "CIDs = ${COLLECTION_IDS[*]}"
+  else
+    echo "No IDs detected in the response."
+  fi
+  HTTP_RETURN_CODE=$(echo echo $1 | tail -1 | sed -e 's/^.*\([0-9][0-9][0-9]\)$/\1/')
+}
+
+echo "# 1. Create Collection Tests"
+echo "=== Testing Collection Creation ==="
+
+echo "# Test 1.1: Create collection with name only"
+make_request "POST" "/collections" \
+    "-d '{\"name\":\"Test Collection\"}'"
+
+echo "# Test 1.2: Create collection with name and description"
+make_request "POST" "/collections" \
+    "-d '{\"name\":\"Test Collection\",\"description\":\"A test collection\"}'"
+
+echo "# Test 1.3: Create collection with empty name (should fail)"
+make_request "POST" "/collections" \
+    "-d '{\"name\":\"\"}'"
+
+echo "# Test 1.4: Create collection with very long name (edge case)"
+make_request "POST" "/collections" \
+    "-d '{\"name\":\"$(printf 'a%.0s' {1..1000})\"}'"
+
+echo "# 2. List Collections Tests"
+echo "=== Testing Collection Listing ==="
+
+echo "# Test 2.1: List all collections"
+make_request "GET" "/collections" ""
+ALL_COLLECTION_IDS=("${COLLECTION_IDS[@]}")
+
+echo "# Test 2.2: List collections with pagination"
+make_request "GET" "/collections?offset=0&limit=10" ""
+
+echo "# Test 2.3: List collections with invalid pagination (edge cases)"
+make_request "GET" "/collections?offset=-1&limit=10" ""
+make_request "GET" "/collections?offset=0&limit=0" ""
+make_request "GET" "/collections?offset=0&limit=1001" ""
+
+echo "# Test 2.4: List specific collections by IDs"
+# Join array elements with commas
+COMMA_SEPARATED_IDS=$(echo "${ALL_COLLECTION_IDS[*]}" | sed -e "s/ /,/")
+make_request "GET" "/collections?ids=${COMMA_SEPARATED_IDS}" ""
+echo "# 3. Get Collection Tests"
+echo "=== Testing Collection Retrieval ==="
+
+echo "# Test 3.1: Get collection by ID"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}" ""
+
+echo "# Test 3.2: Get non-existent collection"
+make_request "GET" "/collections/00000000-0000-0000-0000-000000000000" ""
+
+echo "# Test 3.3: Get collection with invalid UUID format"
+make_request "GET" "/collections/invalid-uuid" ""
+
+echo "# 4. Update Collection Tests"
+echo "=== Testing Collection Updates ==="
+
+echo "# Test 4.1: Update collection name"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"name\":\"Updated Collection Name\"}'"
+
+echo "# Test 4.2: Update collection description"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"description\":\"Updated description\"}'"
+
+echo "# Test 4.3: Update both name and description"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"name\":\"Updated Name\",\"description\":\"Updated description\"}'"
+
+echo "# Test 4.4: Generate description"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"generate_description\":true}'"
+
+echo "# Test 4.5: Invalid update (both description and generate_description)"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"description\":\"New description\",\"generate_description\":true}'"
+
+echo "# 5. Delete Collection Tests"
+echo "=== Testing Collection Deletion ==="
+
+echo "# Test 5.1: Delete existing collection"
+make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}" ""
+
+echo "# Test 5.2: Delete non-existent collection"
+make_request "DELETE" "/collections/00000000-0000-0000-0000-000000000000" ""
+
+echo "# 6. Document Management Tests"
+echo "=== Testing Document Management ==="
+
+echo "# Test 6.1: Add document to collection"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/documents/456e789a-b12c-34d5-e678-901234567890" ""
+
+echo "# Test 6.2: List documents in collection"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=0&limit=10" ""
+
+echo "# Test 6.3: List documents with invalid pagination"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=-1&limit=10" ""
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=0&limit=1001" ""
+
+echo "# Test 6.4: Remove document from collection"
+make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}/documents/456e789a-b12c-34d5-e678-901234567890" ""
+
+echo "# 7. User Management Tests"
+echo "=== Testing User Management ==="
+
+echo "# Test 7.1: List users in collection"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=0&limit=10" ""
+
+echo "# Test 7.2: List users with invalid pagination"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=-1&limit=10" ""
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=0&limit=1001" ""
+
+echo "# Test 7.3: Add user to collection"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/users/789a012b-c34d-5e6f-a789-012345678901" ""
+
+echo "# Test 7.4: Remove user from collection"
+make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}/users/789a012b-c34d-5e6f-a789-012345678901" ""
+
+echo "# 8. Entity Extraction Tests"
+echo "=== Testing Entity Extraction ==="
+
+echo "# Test 8.1: Extract entities with default settings"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
+    "-d '{\"run_type\":\"RUN\"}'"
+
+echo "# Test 8.2: Extract entities with custom settings"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
+    "-d '{\"run_type\":\"RUN\",\"settings\":{\"model\":\"gpt-4\"}}'"
+
+echo "# Test 8.3: Extract entities without orchestration"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
+    "-d '{\"run_type\":\"RUN\",\"run_with_orchestration\":false}'"
+
+echo "# Test 8.4: Get estimate only"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
+"-d '{\"run_type\":\"ESTIMATE\"}'"

cleverbrag/tests/test_data/America-Horse_with_no_name.txt

@@ -0,0 +1,42 @@
+On the first part of the journey
+I was looking at all the life
+There were plants and birds and rocks and things
+There was sand and hills and rings
+The first thing I met was a fly with a buzz
+And the sky with no clouds
+The heat was hot and the ground was dry
+But the air was full of sound
+
+I've been through the desert on a horse with no name
+It felt good to be out of the rain
+In the desert you can remember your name
+'Cause there ain't no one for to give you no pain
+La, la ...
+
+After two days in the desert sun
+My skin began to turn red
+After three days in the desert fun
+I was looking at a river bed
+And the story it told of a river that flowed
+Made me sad to think it was dead
+
+You see I've been through the desert on a horse with no name
+It felt good to be out of the rain
+In the desert you can remember your name
+'Cause there ain't no one for to give you no pain
+La, la ...
+
+After nine days I let the horse run free
+'Cause the desert had turned to sea
+There were plants and birds and rocks and things
+there was sand and hills and rings
+The ocean is a desert with it's life underground
+And a perfect disguise above
+Under the cities lies a heart made of ground
+But the humans will give no love
+
+You see I've been through the desert on a horse with no name
+It felt good to be out of the rain
+In the desert you can remember your name
+'Cause there ain't no one for to give you no pain
+La, la ...

cleverbrag/tests/test_data/Instituto_Superior_Tecnico.md.json

@@ -0,0 +1 @@
+[{"@id": "data:text/plain;charset=US-ASCII,instituto%20superior%20tecnico", "@type": "http://www.cleverthis.com/university#Organization"}, {"@id": "data:text/plain;charset=US-ASCII,lisbon", "@type": "http://www.cleverthis.com/university#City"}, {"@id": "data:text/plain;charset=US-ASCII,lisbon", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,loures", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,Rogerio%20colaco", "@type": "http://www.cleverthis.com/university#Person"}, {"@id": "data:text/plain;charset=US-ASCII,1049-001%20lisboa", "@type": "data:,"}, {"@id": "data:text/plain;charset=US-ASCII,Instituto%20superior%20tecnico", "@type": "http://www.cleverthis.com/university#University", "http://www.cleverthis.com/university#campus": {"@id": "data:text/plain;charset=US-ASCII,three%20campuses", "@type": "http://www.cleverthis.com/university#Campus"}, "data:,": {"@id": "data:text/plain;charset=US-ASCII,tecnico", "@type": "http://www.cleverthis.com/university#Name"}, "http://www.cleverthis.com/university#country": {"@id": "data:text/plain;charset=US-ASCII,portugal", "@type": "http://www.cleverthis.com/university#Country"}}, {"@id": "data:text/plain;charset=US-ASCII,taguspark", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,oeiras", "@type": "http://www.cleverthis.com/university#City"}, {"@id": "data:text/plain;charset=US-ASCII,alameda", "@type": "http://www.cleverthis.com/university#Campus"}]

cleverbrag/tests/test_documents_router.sh

@@ -0,0 +1,180 @@
+#!/bin/bash
+
+. ./test_shared_functions.sh
+
+JSON_FILE=./test_data/Instituto_Superior_Tecnico.md.json
+HTML_FILE=./test_data/AMERICA-VENTURE_HIGHWAY.html
+MP3_FILE=./test_data/flight_of_the_bumblebee_2.mp3
+JPG_FILE=./test_data/China_Beijing_Summer_Palace2.jpg
+EXCEL_FILE=./test_data/Worldbank-Datasheet-02082022.xlsx
+TEXT_FILE=./test_data/America-Horse_with_no_name.txt
+OPEN_OFFICE_FILE=./test_data/GAR_ARchitecture.odg
+
+echo "# 1. Create Document Tests"
+echo "=== Testing Document Creation ==="
+
+echo "# Test 1.1: Create document with file, use different formats"
+make_request "POST" "/documents" \
+    "-F file=@${TEXT_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${JSON_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_JSON_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${HTML_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_HTML_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${MP3_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_MP3_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${JPG_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_JPG_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${EXCEL_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_EXCEL_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${OPEN_OFFICE_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_OPEN_OFFICE_FILE=${DOCUMENT_ID}
+
+echo "# Test 1.2: Create document with raw text"
+make_request "POST" "/documents" \
+    '-F raw_text="On the first part of the journey
+I was looking at all the life
+There were plants and birds and rocks and things
+There was sand and hills and rings
+The first thing I met was a fly with a buzz
+And the sky with no clouds
+The heat was hot and the ground was dry
+But the air was full of sound"' \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_RAW_TEXT=${DOCUMENT_ID}
+
+echo "# Test 1.3: Create document with chunks"
+make_request "POST" "/documents" \
+    "-F chunks=\"[\\\"I've been through the desert on a horse with no name\\\",\\\"It felt good to be out of the rain\\\"]\"" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_CHUNKS=${DOCUMENT_ID}
+
+echo "# Test 1.4: Create document with metadata"
+make_request "POST" "/documents" \
+    "-F file=@${TEXT_FILE} -F metadata='{\"title\":\"Horse with No Name\",\"author\":\"America\"}'" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_FILE_WITH_METADATA=${DOCUMENT_ID}
+
+echo "# Test 1.5: Create document with custom ingestion config"
+make_request "POST" "/documents" \
+    "-F file=@${TEXT_FILE} -F ingestion_mode=hi-res -F ingestion_config='{\"chunk_size\":1000}'" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_FILE_WITH_CUSTOM_INGEST=${DOCUMENT_ID}
+
+echo "# 2. List Documents Tests"
+echo "=== Testing Document Listing ==="
+
+echo "# Test 2.1: List all documents"
+make_request "GET" "/documents" ""
+
+echo "# Test 2.2: List documents with pagination"
+make_request "GET" "/documents?offset=0&limit=10" ""
+
+echo "# Test 2.3: List specific documents by IDs"
+make_request "GET" "/documents?ids=${DOCUMENT_ID_FROM_CHUNKS},${DOCUMENT_ID_FROM_FILE_WITH_METADATA}" ""
+
+echo "# Test 2.4: List documents with summary embeddings"
+make_request "GET" "/documents?include_summary_embeddings=1" ""
+
+echo "# 3. Get Document Tests"
+echo "=== Testing Document Retrieval ==="
+
+echo "# Test 3.1: Get document by ID"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}" ""
+
+echo "# Test 3.2: Get non-existent document"
+make_request "GET" "/documents/00000000-0000-0000-0000-000000000000" ""
+
+echo "# 4. List Document Chunks Tests"
+echo "=== Testing Document Chunks ==="
+
+echo "# Test 4.1: List chunks with pagination"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/chunks?offset=0&limit=10" ""
+
+echo "# Test 4.2: List chunks with vectors"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/chunks?include_vectors=true" ""
+
+echo "# 5. Download Document Tests"
+echo "=== Testing Document Download ==="
+
+echo "# Test 5.1: Download document"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE}/download" ""
+
+echo "# 6. Delete Document Tests"
+echo "=== Testing Document Deletion ==="
+
+echo "# Test 6.1: Delete document by ID"
+make_request "DELETE" "/documents/${DOCUMENT_ID_FROM_RAW_TEXT}" ""
+
+echo "# Test 6.2: Delete documents by filter"
+make_request "DELETE" "/documents/by-filter" \
+    "-d '{\"document_type\":{\"\$eq\":\"txt\"}}'"
+
+echo "# 7. Document Collections Tests"
+echo "=== Testing Document Collections ==="
+
+echo "# Test 7.1: List document collections"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/collections?offset=0&limit=10" ""
+
+echo "# 8. Entity Extraction Tests"
+echo "=== Testing Entity Extraction ==="
+
+echo "# Test 8.1: Extract entities with default settings"
+make_request "POST" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/extract" \
+    "-d '{\"run_type\":\"RUN\"}'"
+
+echo "# Test 8.2: Extract entities with custom settings"
+make_request "POST" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/extract" \
+    "-d '{\"run_type\":\"RUN\",\"settings\":{\"model\":\"gpt-4\"}}'"
+
+echo "# 9. Entity Retrieval Tests"
+echo "=== Testing Entity Retrieval ==="
+
+echo "# Test 9.1: Get entities with pagination"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/entities?offset=0&limit=10" ""
+
+echo "# Test 9.2: Get entities with embeddings"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/entities?include_embeddings=true" ""
+
+echo "# 10. Relationship Tests"
+echo "=== Testing Relationships ==="
+
+echo "# Test 10.1: Get relationships with pagination"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/relationships?offset=0&limit=10" ""
+
+echo "# Test 10.2: Get relationships with filters"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/relationships?entity_names=person,company&relationship_types=works_for" ""
+
+echo "# 11. Document Search Tests"
+echo "=== Testing Document Search ==="
+
+echo "# Test 11.1: Basic search"
+make_request "POST" "/documents/search" \
+    "-d '{\"query\":\"test query\",\"search_mode\":\"basic\"}'"
+
+echo "# Test 11.2: Advanced search with custom settings"
+make_request "POST" "/documents/search" \
+    "-d '{\"query\":\"test query\",\"search_mode\":\"advanced\",\"search_settings\":{\"filters\":{\"document_type\":\"pdf\"}}}'"
+
+echo "# Test 11.3: Custom search with complex filters"
+make_request "POST" "/documents/search" \
+    "-d '{\"query\":\"test query\",\"search_mode\":\"custom\",\"search_settings\":{\"filters\":{\"created_at\":{\"\$gt\":\"2023-01-01\"}}}}'"

cleverbrag/tests/test_shared_functions.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+#
+# Generates random valid UUID
+#
+generate_uuid() {
+    local N B C='89ab'
+    local RES=''
+    for (( N=0; N < 16; ++N )) do
+        B=$(( $RANDOM % 256 ))
+
+        case $N in
+            6)
+                RES="${RES}$(printf '4%x' $(( B % 16 )))"
+                ;;
+            8)
+                RES="${RES}$(printf '%c%x' ${C:$RANDOM%${#C}:1} $(( B % 16 )))"
+                ;;
+            3 | 5 | 7 | 9)
+                RES="${RES}$(printf '%02x-' $B)"
+                ;;
+            *)
+                RES="${RES}$(printf '%02x' $B)"
+                ;;
+        esac
+    done
+    echo -n $RES
+}
+
+#
+# Extract document_id (if present) and HTTP response code from curl response passed as argument
+#
+extract_document_id() {
+  DOCUMENT_ID=$(echo $1 | grep document_id | tail -1 | sed -e 's/^.*"document_id": *"\([^"]*\)".*$/\1/')
+  HTTP_RETURN_CODE=$(echo echo $1 | tail -1 | sed -e 's/^.*\([0-9][0-9][0-9]\)$/\1/')
+}
+
+# Helper function to make curl requests
+make_request() {
+    local method=$1
+    local endpoint=$2
+    local data=$3
+    local headers=$4
+    if [ "xx${headers}" == "xx" ]; then
+      headers='-H "Content-Type: application/json"'
+    fi
+    # echo "Testing $method $endpoint"
+    # echo "Request data: $data"
+    # echo "Request headers: $headers"
+    CMD="curl -X $method \"$API_BASE$endpoint\" -s  -w \"\n%{http_code}\" -H \"Authorization: Bearer $AUTH_TOKEN\" $headers $data"
+    echo $CMD
+    RES=$(eval $CMD 2>/dev/null)
+    echo "Response: [${RES}]"
+    extract_document_id "$RES"
+    # echo "document_id --> ${DOCUMENT_ID}"
+    if [[ ${HTTP_RETURN_CODE} -lt 300 ]]; then
+      echo "SUCCESS. code = ${HTTP_RETURN_CODE}"
+    else
+      echo "ERROR. code = ${HTTP_RETURN_CODE}"
+    fi
+    echo -e "\n\n"
+    if [ "${EXECUTE_ALL}" != "-a" ]; then
+      echo "Press any key to continue..."
+      read -n 1 -s -r
+      echo -e "\n\n"
+    fi
+}
+
+#
+# Simple scanner of cmd line arguments, inferring meaning of the value
+# It recognized:
+#  a)  host URL (starts with 'http' prefix)
+#  b)  JWT token (starts with 'ey' prefix)
+#  c)  flag -a  to execute all tests w/o waiting for user pressing any key
+#
+while [ $# -gt 0 ]; do
+    echo "Argument: [${1}]"
+    if [[ "$1" == ey* ]]; then
+      AUTH_TOKEN=$1
+    fi
+    if [[ "$1" == http* ]]; then
+      HOST=$1
+    fi
+    if [[ "$1" == "-a" ]]; then
+      EXECUTE_ALL=$1
+    fi
+    shift  # Removes $1 and shifts remaining args left
+done
+
+
+if [ "xx${HOST}" == "xx" ]; then
+  HOST="http://localhost:7272"
+fi
+API_BASE="$HOST/v3"
+# Once integrated with CleverMicro, token can be obtained from CleverMicro
+# if [ "xx${AUTH_TOKEN}" == "xx" ]; then
+#  AUTH_TOKEN=$(curl $HOST/api/v0/login -s -d '{"username":"hugo@cleverthis.com","password":"tellmeall"}' -H "Content-Type: application/json" | sed -e "s/.*access_token...\(.*\)..$/\1/")
+# fi

cleverswarm/clever_swarm_adapter.py

@@ -0,0 +1,81 @@
+"""
+This module provides the CleverSwarm AMQP Adapter for handling API requests via AMQP.
+The code IS MEANT to be part of CleverSwarm codebase, and has direct dependencies on the CleverSwarm codebase.
+It is NOT MEANT to be used as a standalone module. It WILL NOT run outside the CleverSwarm endpoints context.
+"""
+
+import importlib
+from threading import Thread
+from typing import Any, List, Optional
+
+# CleverSwarm specific imports
+import core.deps
+from fastapi.routing import APIRoute
+from schemas.user_schema import UserSchema
+
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.service.amq_service import AMQService
+
+PREFERRED_SUFFIX = "_json"
+
+
+# =================================================================================================
+# =========================== S u p p o r t i n g   f u n c t i o n s =============================
+# =================================================================================================
+def get_callable_for_name(module_name: str, function_name: str) -> Optional[callable]:
+    """
+    Get the function/Callable for the module_name.function_name if it exists in the module, else return None.
+    """
+    try:
+        _module = importlib.import_module(module_name)
+        _preferred_endpoint = getattr(_module, function_name)
+        return _preferred_endpoint if callable(_preferred_endpoint) else None
+    except Exception:
+        return None
+
+
+# =================================================================================================
+# ======================== C l e v e r S w a r m   A M Q    A d a p t e r  ========================
+# =================================================================================================
+class CleverSwarmAmqpAdapter(CleverThisServiceAdapter):
+    """
+    CleverSwarm AMQP Adapter for CleverSwarm API. Implements the CleverSwarm specific logic that overrides
+    common logic defined by AMQ Adapter library. The specific logic here is:
+
+    1. Extract subject from JWT token and convert it to UserSchema to avail authorized user argument
+    2. Find JSON wrapper for endpoint Callable, and if exists, use it instead of the original endpoint.
+
+    """
+
+    def __init__(self, amq_configuration: AMQConfiguration, routes: List[APIRoute]):
+        super().__init__(amq_configuration, routes=routes, session=None)
+        _amq_service: AMQService = AMQService(amq_configuration, self)
+        self.thread = Thread(target=_amq_service.run)
+        self.thread.start()
+
+    async def get_current_user(self, token: str) -> UserSchema:
+        """
+        Overrides the default get_current_user method to use the token from the AMQP message.
+        In CleverSwarm context, it means simply to call provided get_current_user function.
+
+        :param token: JWT token from the AMQP message
+
+        :return: UserSchema object
+        """
+        return await core.deps.get_current_user(token)
+
+    def get_endpoint(self, endpoint: Any) -> Any:
+        """
+        If the specific endpoint has a matching JSON wrapper (identified as {endpoint.name}_json ),
+        then use it as preferred way to invoke the endpoint.
+
+        :param endpoint: the Callable for the endpoint
+
+        :return: new Callable representing the JSON wrapper endpoint, if it exists or the original Callable.
+        """
+        _preferred_endpoint = get_callable_for_name(
+            getattr(endpoint, "__module__"),
+            endpoint.__name__ + PREFERRED_SUFFIX,
+        )
+        return _preferred_endpoint if _preferred_endpoint is not None else endpoint

cleverswarm/main.py

@@ -0,0 +1,8 @@
+from clever_swarm_adapter import CleverSwarmAmqpAdapter
+
+from amqp.config.amq_configuration import AMQConfiguration
+
+if __name__ == "__main__":
+    adapter = CleverSwarmAmqpAdapter(AMQConfiguration("./application.properties.local"))
+    print("Press ^C to exit...")
+    adapter.thread.join()

compose.yaml

@@ -0,0 +1,244 @@
+name: clever_micro
+services:
+
+  rabbitmq: # copied from cleverdata backend branch feat/#190
+    image: rabbitmq:3-management
+    hostname: "my-rabbit"
+    environment:
+      RABBITMQ_DEFAULT_USER: springuser
+      RABBITMQ_DEFAULT_PASS: TheCleverWho
+    volumes:
+      - "./container-data/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins"
+    #      - "./container-data/mq-data:/var/lib/rabbitmq"
+    ports:
+      - "5672:5672"
+      - "8888:15672"
+    healthcheck:
+      test: ["CMD-SHELL", "rabbitmq-diagnostics -q ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  traefik:
+    image: "traefik:v3.2"
+    container_name: "traefik"
+    command:
+      - "--api.insecure=true"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entryPoints.web.address=:8085"
+    labels:
+      - "traefik.enable=true"
+    ports:
+      - "8085:8085"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+
+  jaeger:
+    image: "jaegertracing/all-in-one:latest"
+    ports:
+      - "16686:16686"  # Jaeger UI
+      - "14268:14268"  # Jaeger collector Accepts jaeger.thrift spans directly over HTTP
+      - "14250:14250"  # Jaeger gRPC Accepts model.proto spans over gRPC
+      - "6831:6831/udp"  # Jaeger agent Accepts jaeger.thrift spans (Thrift compact)
+      - "6832:6832/udp"  # Jaeger agent Accepts jaeger.thrift spans (Thrift binary)
+      - "5778:5778"    # Jaeger agent admin + configuration
+      - "4317:4317"    # OpenTelemetry Protocol (OTLP) gRPC receiver
+      - "4318:4318"    # OpenTelemetry Protocol (OTLP) HTTP/protobuf receiver
+      - "14269:14269"  # Jaeger health check
+      - "9411:9411"    # Zipkin compatibility
+    environment:
+      COLLECTOR_ZIPKIN_HTTP_PORT: 9411
+      COLLECTOR_OTLP_ENABLED: true
+
+  amqp-router:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    #  dockerfile: Dockerfile.amqp-adapter
+    #image: "amq-python:latest"
+    image: "amqp_router:latest"
+    ports:
+      - "${AMQ_ADAPTER_PORT:-8081}:8080"
+    environment:
+      cm.amq-adapter.service-name: "amq-router"
+      cm.amq-adapter.is-router: "true"
+      cm.amq-adapter.generator-id: 164
+      cm.amq-adapter.route-mapping: "null"
+
+      cm.dispatch.use-dlq: true
+      cm.dispatch.use-confirms: false
+      cm.dispatch.amq-host: rabbitmq
+      cm.dispatch.amq-port: 5672
+      cm.dispatch.amq-port-tls: 5671
+
+      cm.backpressure.threshold: 20
+      cm.backpressure.time-window: 3000
+      cm.backpressure.management-service-url: "management-service:8080"
+      quarkus.otel.exporter.otlp.endpoint: "http://jaeger:4317"
+      quarkus.application.name: API-Gateway-AMQ-Adapter
+      quarkus.otel.service.name: API-Gateway-AMQ-Adapter
+    env_file:
+      - .env_secrets
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.router-app.rule=PathPrefix(`/`)"  # default route
+      - "traefik.http.routers.router-app.entrypoints=web"
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "curl http://localhost:8080/amq-adapter-healthcheck"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  #
+  # DLQ is special case of AMQ adapter that will handle rejected messages
+  #
+  dead-letter-queue-handler:
+    image: "amqp_router:latest"
+    ports:
+      - "${DLQ_ADAPTER_PORT:-8089}:8080"
+    environment:
+
+      cm.amq-adapter.service-name: "amq-dlq"
+      cm.amq-adapter.is-router: "false"
+      cm.amq-adapter.generator-id: 165
+      cm.amq-adapter.route-mapping: "amq-dlq:DLX:DeadLetterQueue:#:0:0"
+
+      cm.dispatch.use-dlq: false
+      cm.dispatch.use-confirms: false
+      cm.dispatch.amq-host: rabbitmq
+      cm.dispatch.amq-port: 5672
+      cm.dispatch.amq-port-tls: 5671
+
+      cm.backpressure.threshold: 20
+      cm.backpressure.time-window: 3000
+      cm.backpressure.management-service-url: "management-service:8080"
+      quarkus.application.name: AMQ-DeadLetterQueue
+      quarkus.otel.exporter.otlp.endpoint: "http://jaeger:4317"
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+    env_file:
+      - .env_secrets
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+  #
+  # test applications
+  #
+  amq-clever-book:
+    image: "amqp_router:latest"
+    ports:
+      - "16000:8080"
+    environment:
+
+      cm.amq-adapter.service-name: "amq-cleverbook-adapter"
+      cm.amq-adapter.generator-id: 16
+      cm.amq-adapter.route-mapping: "amq-clever-book::amq-clever-book:clever.book.localhost.#:5:0"
+
+      cm.dispatch.use-dlq: true
+      cm.dispatch.use-confirms: false
+      cm.dispatch.amq-host: rabbitmq
+      cm.dispatch.amq-port: 5672
+      cm.dispatch.amq-port-tls: 5671
+      cm.dispatch.service-host: clever-book
+      cm.dispatch.service-port: 8080
+
+      cm.backpressure.threshold: 20
+      cm.backpressure.time-window: 3000
+      cm.backpressure.management-service-url: "management-service:8080"
+      quarkus.otel.service.name: CleverBook-AMQ-Adapter
+      quarkus.application.name: CleverBook-AMQ-Adapter
+      quarkus.otel.exporter.otlp.endpoint: "http://jaeger:4317"
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+    env_file:
+      - .env_secrets
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+
+  #
+  # Dummy app allowing POST and GET operations over REST API, emulating a real SpringBoot service
+  #
+  clever-book:
+    build:
+      context: clever-book/
+      dockerfile: Dockerfile
+    image: "book-management:latest"
+    ports:
+      - "16002:8080"
+    restart: on-failure
+    environment:
+      SPRING_PROFILES_ACTIVE: prod
+      SPRING_APPLICATION_NAME: CleverBook
+      spring.application.name: CleverBook
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+      otel.exporter.otlp.endpoint: "http://jaeger:4317"
+    env_file:
+      - .env_secrets
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+
+  #
+  # a black box 3rd party app provisioned as-is
+  #
+  black-box-3rd-party:
+    image: "traefik/whoami"
+    container_name: "blackbox-third-party"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.whoami.rule=Host(`whoami.localhost`)"
+      - "traefik.http.routers.whoami.entrypoints=web"
+
+  #
+  # test applications for python
+  #
+  amq-clever-pybook:
+    image: "amq-python:latest"
+    # image: "amqp_router:latest"
+    ports:
+      - "16020:8080"
+    environment:
+
+      cm.amq-adapter.service-name: "amq-cleverpybook-adapter"
+      cm.amq-adapter.generator-id: 211
+      cm.amq-adapter.route-mapping: "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+
+      cm.dispatch.use-dlq: true
+      cm.dispatch.use-confirms: false
+      cm.dispatch.amq-host: rabbitmq
+      cm.dispatch.amq-port: 5672
+      cm.dispatch.amq-port-tls: 5671
+      cm.dispatch.service-host: clever-pybook
+      cm.dispatch.service-port: 5000
+
+      cm.backpressure.threshold: 20
+      cm.backpressure.time-window: 3000
+      cm.backpressure.management-service-url: "management-service:8080"
+      quarkus.otel.service.name: CleverPyBook-AMQ-Adapter
+      quarkus.application.name: CleverPyBook-AMQ-Adapter
+      quarkus.otel.exporter.otlp.endpoint: "http://jaeger:4317"
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+    env_file:
+      - .env_secrets
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+
+  #
+  # Dummy app allowing POST and GET operations over REST API, emulating a real SpringBoot service
+  #
+  clever-pybook:
+    image: "clever-pybook:latest"
+    ports:
+      - "16012:5000"
+    restart: on-failure
+    environment:
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
+      OTEL_EXPORTER_OTLP_INSECURE: "true"
+    depends_on:
+      amqp-router:
+        condition: service_healthy
+

container-data/alertmanager/alertmanager.yml

@@ -0,0 +1,16 @@
+route:
+  receiver: 'demo.web.hook'
+  group_by: ['path', 'queue']
+  group_wait: 3s
+  group_interval: 1s
+  repeat_interval: 1m
+receivers:
+  - name: 'demo.web.hook'
+    webhook_configs:
+      - url: 'http://10.233.1.4:8080/alert'
+inhibit_rules:
+  # override warnings if we have critical
+  - source_match:
+      severity: 'critical'
+    target_match:
+      severity: 'warning'

container-data/grafana/dashboard-export/Application-1730708374720.json

@@ -0,0 +1,271 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "11.3.0"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.3.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "rabbitmq_queue_messages_ready{queue=\"demo.getRandom\"}",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false,
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          }
+        }
+      ],
+      "title": "RabbitMQ Queue getRandom",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.3.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "delta(application_demo_http_req_duration_second_seconds_count[$__interval])",
+          "fullMetaSearch": false,
+          "includeNullMetadata": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "/number req",
+      "type": "timeseries"
+    }
+  ],
+  "schemaVersion": 40,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Application",
+  "uid": "ce2wmjrfkm7lsf",
+  "version": 5,
+  "weekStart": ""
+}

container-data/kibana/config/kibana.yml

@@ -0,0 +1,9 @@
+server.host: 0.0.0.0
+elasticsearch.hosts: [ http://elasticsearch:9200 ]
+
+monitoring.ui.container.elasticsearch.enabled: true
+monitoring.ui.container.logstash.enabled: true
+
+# X-Pack security credentials
+elasticsearch.username: kibana_system
+elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}

container-data/logstash/pipeline/logstash.conf

@@ -0,0 +1,15 @@
+input {
+	beats {
+		port => 5044
+	}
+}
+
+## Add your filters / logstash plugins configuration here
+
+output {
+	elasticsearch {
+		hosts => "elasticsearch:9200"
+		user => "logstash_internal"
+		password => "${LOGSTASH_INTERNAL_PASSWORD}"
+	}
+}

container-data/prometheus/alert-rules.yml

@@ -0,0 +1,21 @@
+groups:
+  - name: demo_application
+    rules:
+      - alert: application_http_duration_alert
+        expr: rate(application_demo_http_req_duration_second_seconds_bucket{le="+Inf"}[5s]) >= 100
+        for: 5s
+        labels:
+          severity: critical
+        annotations:
+          summary: "Application process http req take too long"
+          description: "Endpoint {{ $labels.path }} is taking too long"
+  - name: rabbitmq
+    rules:
+      - alert: rabbitmq_queue_message_ready_count_alert
+        expr: rabbitmq_queue_messages_ready >= 50
+        for: 3s
+        labels:
+          severity: critical
+        annotations:
+          summary: "RabbitMQ queue backpressure too high"
+          description: "Queue {{ $labels.queue }} is taking too long"

container-data/prometheus/prometheus.yml

@@ -0,0 +1,32 @@
+scrape_configs:
+  - job_name: RabbitMQ
+    scrape_interval: 2s
+    metrics_path: /metrics/per-object
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+    relabel_configs:
+      # Only keep containers that have a `prometheus-job=rabbitmq` label.
+      # also requires the port = 15962, that's the report port of RabbitMQ
+      - source_labels: [__meta_docker_container_label_prometheus_job, __meta_docker_port_private]
+        regex: rabbitmq;15692
+        action: keep
+
+  - job_name: SpringActuator
+    metrics_path: /actuator/prometheus
+    scrape_interval: 2s
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+    relabel_configs:
+      - source_labels: [__meta_docker_container_label_prometheus_job, __meta_docker_port_private]
+        regex: spring-actuator;8080
+        action: keep
+
+alerting:
+  alertmanagers:
+    - scheme: http
+      static_configs:
+        - targets:
+          - "alertmanager:9093"
+
+rule_files:
+  - "alert-rules.yml"

container-data/rabbitmq/enabled_plugins

@@ -0,0 +1 @@
+[rabbitmq_management,rabbitmq_prometheus].

container-data/traefik/traefik.yml

@@ -0,0 +1,20 @@
+api:
+  insecure: true
+  dashboard: true
+  debug: true
+  disableDashboardAd: true
+providers:
+  docker:
+    exposedbydefault: false
+log:
+  level: "DEBUG"
+
+entryPoints:
+  web:
+    address: ":8080"
+
+#  ampqs:
+#    address: ":3333"
+
+#  websecure:
+#    address: ":443"

demo.py

@@ -0,0 +1,336 @@
+import json
+import os
+import random
+import shutil
+import time
+
+import httpx  # For emulating remote service call
+
+# import uvicorn
+from fastapi import FastAPI, File, Form, UploadFile
+
+# OpenTelemetry imports
+from opentelemetry import metrics, trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.exporter.prometheus import PrometheusMetricReader
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+from opentelemetry.sdk.metrics import MeterProvider
+
+# from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+
+from amqp.config.amq_configuration import AMQConfiguration
+from otdemo.otdemo_adapter import OTDemoAmqpAdapter
+
+# --- OpenTelemetry Configuration ---
+
+# Configure Resource: Defines common attributes for traces and metrics
+resource = Resource.create(
+    attributes={
+        "service.name": "fastapi-document-service",
+        "application": "CleverMicroDemo",
+        "environment": "development",
+    }
+)
+
+# Configure TracerProvider: Responsible for creating and managing Tracers
+trace_provider = TracerProvider(resource=resource)
+
+# OTLPSpanExporter: Exports spans to an OTLP collector (e.g., Jaeger) via gRPC
+# Endpoint can be overridden by OTEL_EXPORTER_OTLP_ENDPOINT environment variable
+otlp_exporter = OTLPSpanExporter(
+    endpoint=os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")
+)
+# BatchSpanProcessor: Batches spans for efficient export
+trace_provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
+
+# Optional: ConsoleSpanExporter for local debugging to print traces to console
+if os.environ.get("OTEL_DEBUG_CONSOLE_TRACES", "false").lower() == "true":
+    trace_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+
+# Set the configured TracerProvider globally
+trace.set_tracer_provider(trace_provider)
+
+# Configure MeterProvider: Responsible for creating and managing Meters for metrics
+# PrometheusMetricReader: Exposes metrics via an HTTP endpoint for Prometheus to scrape
+# Host and port can be overridden by OTEL_EXPORTER_PROMETHEUS_HOST and OTEL_EXPORTER_PROMETHEUS_PORT
+prometheus_reader = PrometheusMetricReader(
+    #    handler_address=os.environ.get("OTEL_EXPORTER_PROMETHEUS_HOST", "0.0.0.0"),
+    #    handler_port=int(os.environ.get("OTEL_EXPORTER_PROMETHEUS_PORT", "8001"))
+)
+# MeterProvider: Uses the PrometheusMetricReader to periodically export metrics
+metric_provider = MeterProvider(resource=resource, metric_readers=[prometheus_reader])
+# Set the configured MeterProvider globally
+metrics.set_meter_provider(metric_provider)
+
+# Get tracer and meter instances from the global providers
+tracer = trace.get_tracer("document-service.tracer")
+meter = metrics.get_meter("document-service.meter")
+
+# --- Custom Metrics Definition ---
+# Counters: For cumulative sums (e.g., total requests)
+documents_uploaded_counter = meter.create_counter(
+    name="documents_uploaded_total", description="Total number of documents uploaded", unit="1"
+)
+metadata_processed_counter = meter.create_counter(
+    name="documents_metadata_processed_total",
+    description="Total number of document metadata processed",
+    unit="1",
+)
+documents_retrieved_counter = meter.create_counter(
+    name="documents_retrieved_total", description="Total number of documents retrieved", unit="1"
+)
+# Histograms: For distributions of values (e.g., request durations)
+internal_api_call_duration_histogram = meter.create_histogram(
+    name="internal_api_call_duration_seconds",
+    description="Duration of internal API calls to metadata endpoint",
+    unit="s",
+)
+
+# --- FastAPI Application Setup ---
+app = FastAPI(
+    title="CleverMicro Document Service Demo",
+    description="A FastAPI application demonstrating OpenTelemetry integration with Jaeger and Prometheus.",
+)
+
+# Instrument FastAPI: Automatically creates spans for incoming requests
+FastAPIInstrumentor.instrument_app(app)
+# Instrument httpx: Automatically propagates trace context for outgoing HTTP calls
+HTTPXClientInstrumentor().instrument()
+
+# Directory for storing uploaded files (temporary for demo)
+UPLOAD_DIR = "/tmp/clevermicro_documents"
+os.makedirs(UPLOAD_DIR, exist_ok=True)
+
+otadapter = OTDemoAmqpAdapter(AMQConfiguration("otdemo/otdemo.properties"), app.routes)
+
+
+@app.post("/api/v3/documents")
+async def upload_document(
+    file: UploadFile = File(...),  # Multipart file upload
+    name: str = Form(...),  # Form field for document name
+    description: str = Form(...),  # Form field for document description
+):
+    # Create a custom span for this specific endpoint's logic
+    with tracer.start_as_current_span("upload_document_endpoint_logic"):
+        # Save the uploaded file to the temporary directory
+        file_location = os.path.join(UPLOAD_DIR, file.filename)
+        try:
+            with open(file_location, "wb") as buffer:
+                shutil.copyfileobj(file.file, buffer)
+            print(f"File '{file.filename}' saved to {file_location}")
+
+            # Increment the documents_uploaded_total metric
+            documents_uploaded_counter.add(1, {"file.name": file.filename, "document.name": name})
+
+            # Emulate a REST call to a "remote" service (which is actually this same app)
+            # This demonstrates trace context propagation across HTTP calls.
+            metadata_payload = {"name": name, "description": description}
+            print(
+                f"Emulating remote call to /api/v3/documents/metadata with: {json.dumps(metadata_payload)}"
+            )
+
+            start_time = time.time()
+            async with httpx.AsyncClient() as client:
+                # httpx instrumentation ensures the current trace context is propagated
+                response = await client.post(
+                    # IMPORTANT: Use the actual host and port where your FastAPI app is running
+                    # If running directly on host, use http://localhost:8000
+                    # If running inside Docker and calling itself, use http://host.docker.internal:8000
+                    # "http://localhost:8000/api/v3/documents/metadata",
+                    "http://host.docker.internal:8000/api/v3/documents/metadata",
+                    json=metadata_payload,
+                )
+                response.raise_for_status()  # Raise an exception for bad status codes (4xx or 5xx)
+            end_time = time.time()
+            duration = end_time - start_time
+            # Record the duration of the internal API call
+            internal_api_call_duration_histogram.record(
+                duration, {"endpoint": "/api/v3/documents/metadata"}
+            )
+
+            print(f"Internal metadata service responded: {response.json()}")
+
+            return {
+                "message": "Document uploaded and metadata processing initiated",
+                "filename": file.filename,
+                "name": name,
+                "description": description,
+            }
+        except httpx.HTTPStatusError as e:
+            print(f"Error calling metadata service: {e.response.status_code} - {e.response.text}")
+            # Propagate the error in the trace
+            current_span = trace.get_current_span()
+            current_span.set_attribute("error.type", "HTTPStatusError")
+            current_span.record_exception(e)
+            return {
+                "message": "Document uploaded but metadata processing failed",
+                "detail": str(e),
+            }, 500
+        except Exception as e:
+            print(f"An unexpected error occurred during document upload: {e}")
+            # Propagate the error in the trace
+            current_span = trace.get_current_span()
+            current_span.set_attribute("error.type", "UnexpectedError")
+            current_span.record_exception(e)
+            return {"message": "Error processing document", "detail": str(e)}, 500
+
+
+@app.post("/api/v3/documents/metadata")
+async def process_document_metadata(metadata: dict):
+    # This endpoint is automatically instrumented by FastAPIInstrumentor
+    # A new span will be created, and its parent will be the span from the calling /api/v3/documents
+    # due to trace context propagation via httpx.
+    print(f"Received metadata for processing: {json.dumps(metadata, indent=2)}")
+
+    # Increment the metadata_processed_total metric
+    metadata_processed_counter.add(1, {"document.name": metadata.get("name")})
+
+    # Simulate some processing time to make traces more interesting
+    time.sleep(random.uniform(0.05, 0.2))
+
+    return {"status": "metadata processed", "received_data": metadata}
+
+
+@app.get("/api/v3/documents")
+async def get_documents():
+    # Create a custom span for the logic within this endpoint
+    with tracer.start_as_current_span("get_documents_endpoint_logic"):
+        documents = []
+        # Generate several random documents to emulate a list
+        for i in range(random.randint(2, 5)):
+            documents.append(
+                {
+                    "id": f"doc-{random.randint(1000, 9999)}",
+                    "name": f"Document {random.randint(1, 100)}",
+                    "description": f"Description for document {random.randint(1, 1000)}",
+                    "uploaded_at": time.strftime(
+                        "%Y-%m-%dT%H:%M:%SZ",
+                        time.gmtime(time.time() - random.randint(0, 86400 * 30)),
+                    ),
+                }
+            )
+
+        # Increment the documents_retrieved_total metric
+        documents_retrieved_counter.add(1)
+
+        # Simulate some processing time
+        time.sleep(random.uniform(0.01, 0.1))
+
+        return {"documents": documents}
+
+
+# --- How to Run the Demo ---
+# 1. Save the code:
+#    Save the Python code above as `main.py` in a new directory.
+
+# 2. Create `requirements.txt`:
+#    Create a file named `requirements.txt` in the same directory with the following content:
+#    ```
+#    fastapi==0.111.0
+#    uvicorn[standard]==0.29.0
+#    python-multipart==0.0.9
+#    httpx==0.27.0
+#    opentelemetry-api==1.25.0
+#    opentelemetry-sdk==1.25.0
+#    opentelemetry-exporter-otlp-proto-grpc==1.25.0 # For Jaeger traces
+#    opentelemetry-exporter-prometheus==1.25.0 # For Prometheus metrics
+#    opentelemetry-instrumentation-fastapi==0.46b0 # IMPORTANT: Check compatibility with your FastAPI version!
+#    opentelemetry-instrumentation-httpx==0.46b0 # IMPORTANT: Check compatibility with your httpx version!
+#    protobuf==4.25.3 # Required by OTLP exporter, ensure version compatibility if issues arise
+#    ```
+
+# 3. Install dependencies:
+#    Open your terminal in the directory where you saved `main.py` and `requirements.txt`, then run:
+#    `pip install -r requirements.txt`
+
+# 4. Run Jaeger and Prometheus (using Docker Compose for simplicity):
+#    Create a file named `docker-compose.yaml` in the same directory:
+#    ```yaml
+#    version: '3.8'
+#    services:
+#      jaeger:
+#        image: jaegertracing/all-in-one:latest
+#        ports:
+#          - "6831:6831/udp" # UDP Thrift
+#          - "14268:14268" # HTTP Thrift
+#          - "14250:14250" # gRPC
+#          - "4317:4317" # OTLP gRPC collector (for traces from our app)
+#          - "4318:4318" # OTLP HTTP collector
+#          - "16686:16686" # Jaeger UI
+#        environment:
+#          - COLLECTOR_OTLP_ENABLED=true # Enable OTLP reception
+#
+#      prometheus:
+#        image: prom/prometheus:latest
+#        volumes:
+#          - ./prometheus.yml:/etc/prometheus/prometheus.yml # Mount Prometheus config
+#        ports:
+#          - "9090:9090" # Prometheus UI
+#        command:
+#          - '--config.file=/etc/prometheus/prometheus.yml'
+#    ```
+
+#    Create a file named `prometheus.yml` in the same directory:
+#    ```yaml
+#    global:
+#      scrape_interval: 15s # How frequently to scrape targets
+#
+#    scrape_configs:
+#      - job_name: 'fastapi_app'
+#        # The 'metrics' endpoint of our FastAPI app exposed by PrometheusMetricReader
+#        # 'host.docker.internal' allows Docker containers to connect to the host machine's localhost
+#        static_configs:
+#          - targets: ['host.docker.internal:8001']
+#    ```
+
+#    Start Jaeger and Prometheus:
+#    `docker-compose up -d` (the `-d` runs them in the background)
+
+# 5. Run the FastAPI application:
+#    In your terminal (where `main.py` is located), run:
+#    `uvicorn main:app --host 0.0.0.0 --port 8000`
+#    (Optional: To see traces printed to console, set `export OTEL_DEBUG_CONSOLE_TRACES=true` before `uvicorn`)
+
+# --- Accessing the UIs ---
+# * **Jaeger UI:** Open your web browser and go to `http://localhost:16686`
+# * **Prometheus UI:** Open your web browser and go to `http://localhost:9090`
+# * **FastAPI App (Root):** `http://localhost:8000`
+# * **FastAPI Metrics Endpoint:** `http://localhost:8001/metrics` (Prometheus scrapes this, you can also view it directly)
+
+# --- How to Test and Observe ---
+# 1. Ensure all services are running (`docker-compose ps` should show `jaeger` and `prometheus` up, and `uvicorn` running in your terminal).
+# 2. **Trigger GET requests:**
+#    Open your browser or use `curl` to hit: `http://localhost:8000/api/v3/documents`
+#    Refresh a few times.
+# 3. **Trigger POST requests (for file upload and internal call):**
+#    You'll need a tool like Postman, Insomnia, or `curl` (more complex for multipart) for this.
+#    **Using Postman/Insomnia:**
+#    * Method: `POST`
+#    * URL: `http://localhost:8000/api/v3/documents`
+#    * Body: Select `form-data`
+#        * Add a key `file`, Type `File`, Value: Choose any small file from your computer (e.g., a `.txt` or `.png`).
+#        * Add a key `name`, Type `Text`, Value: `My Demo Document`
+#        * Add a key `description`, Type `Text`, Value: `A test document for OpenTelemetry.`
+#    * Send the request multiple times.
+
+# 4. **Observe in Jaeger UI (`http://localhost:16686`):**
+#    * Select "Service": `fastapi-document-service`
+#    * Click "Find Traces".
+#    * You should see traces for `/api/v3/documents` (POST) and `/api/v3/documents` (GET).
+#    * Crucially, for POST requests, expand the trace: you will see the main `/api/v3/documents` span, and nested within it, a child span for the `POST /api/v3/documents/metadata` HTTP request. This demonstrates automatic context propagation.
+
+# 5. **Observe in Prometheus UI (`http://localhost:9090`):**
+#    * Go to the "Graph" tab.
+#    * In the expression bar, type and execute queries like:
+#        * `documents_uploaded_total`
+#        * `documents_metadata_processed_total`
+#        * `documents_retrieved_total`
+#        * `internal_api_call_duration_seconds_sum`
+#        * `internal_api_call_duration_seconds_count`
+#    * You will see the values of these metrics increasing with your requests.
+
+# This setup provides a clear, runnable demo for OpenTelemetry traces and metrics in a FastAPI application, including automatic context propagation for internal HTTP calls.

otdemo/otdemo.properties

@@ -0,0 +1,41 @@
+# ====================================================================
+# CleverMicro AMQ Adapter settings
+# ====================================================================
+[CleverMicro-AMQ]
+# CleverMicro AMQ Adapter settings that identify this AMQ Adapter instance to other adapters
+
+# A name to identify this service for the AMQ Adapter
+cm.amq-adapter.service-name=otdemo
+# The CleverMicro AMQ Adapter instance ID, used to identify this instance in the CleverMicro AMQ Adapter cluster
+cm.amq-adapter.generator-id=1
+# Route mapping. For route detail description see section 3.1 in Onboarding Guide for Python:
+# https://docs.cleverthis.com/en/architecture/microservices/onboarding/python
+# NOTE: this is a comma-separated list of route mappings (multiple, comma-separated routes are allowed here)
+cm.amq-adapter.route-mapping=otdemo::otdemo:otdemo.localhost.#:5:0
+# Indicate if AMQ Adapter should respond with HTTP 401 Unauthorized if the request does not contain valid JWT token
+cm.amq-adapter.require-authenticated-user=false
+
+# CleverMicro AMQ Adapter dispatcher, settings related to RabbitMQ connection and optional REST forwarding
+cm.dispatch.use-dlq=true
+cm.dispatch.amq-host=rabbitmq
+cm.dispatch.amq-port=5672
+cm.dispatch.amq-port-tls=5671
+cm.dispatch.download-dir=/tmp/downloaded_files
+# Not used for CleveSwarm nor CleverBRAG or in tight coupling mode.
+# applicable only in loose coupling mode, specify the destination where to forward the REST request
+#cm.dispatch.service-host=cleverthis-service-name.localhost
+#cm.dispatch.service-port=8080
+
+# Backpressure monitor settings
+# Define maximum number of messages of being processed in parallel before triggering backpressure OVERLOAD alert
+cm.backpressure.threshold=5
+# Define maximum CPU usage (%) before triggering backpressure OVERLOAD alert
+cm.backpressure.threshold-cpu-overload=90
+# Define maximum CPU usage (%) before triggering backpressure IDLE alert
+cm.backpressure.threshold-cpu-idle=10
+# Define the time window between the Backpressure reports, in seconds
+cm.backpressure.time-window=10
+# Define the time window length, in seconds, during which, if CPU usage is consistently above the threshold, the system will trigger OVERLOAD alert
+cm.backpressure.cpu-overload-duration=30
+# Define the time window length, in seconds, during which, if no request is made to the Service, the system will trigger IDLE alert
+cm.backpressure.cpu-idle-duration=30

otdemo/otdemo_adapter.py

@@ -0,0 +1,59 @@
+"""
+This module provides the CleverSwarm AMQP Adapter for handling API requests via AMQP.
+The code IS MEANT to be part of CleverSwarm codebase, and has direct dependencies on the CleverSwarm codebase.
+It is NOT MEANT to be used as a standalone module. It WILL NOT run outside the CleverSwarm endpoints context.
+"""
+
+from threading import Thread
+from typing import Any, List
+
+# CleverSwarm specific imports
+from fastapi.routing import APIRoute
+
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.service.amq_service import AMQService
+
+PREFERRED_SUFFIX = "_json"
+
+
+# =================================================================================================
+# ======================== C l e v e r S w a r m   A M Q    A d a p t e r  ========================
+# =================================================================================================
+class OTDemoAmqpAdapter(CleverThisServiceAdapter):
+    """
+    CleverSwarm AMQP Adapter for CleverSwarm API. Implements the CleverSwarm specific logic that overrides
+    common logic defined by AMQ Adapter library. The specific logic here is:
+
+    1. Extract subject from JWT token and convert it to UserSchema to avail authorized user argument
+    2. Find JSON wrapper for endpoint Callable, and if exists, use it instead of the original endpoint.
+
+    """
+
+    def __init__(self, amq_configuration: AMQConfiguration, routes: List[APIRoute]):
+        super().__init__(amq_configuration, routes=routes, session=None)
+        _amq_service: AMQService = AMQService(amq_configuration, self)
+        self.thread = Thread(target=_amq_service.run)
+        self.thread.start()
+
+    async def get_current_user(self, token: str) -> str:
+        """
+        Overrides the default get_current_user method to use the token from the AMQP message.
+        In CleverSwarm context, it means simply to call provided get_current_user function.
+
+        :param token: JWT token from the AMQP message
+
+        :return: UserSchema object
+        """
+        return token
+
+    def get_endpoint(self, endpoint: Any) -> Any:
+        """
+        If the specific endpoint has a matching JSON wrapper (identified as {endpoint.name}_json ),
+        then use it as preferred way to invoke the endpoint.
+
+        :param endpoint: the Callable for the endpoint
+
+        :return: new Callable representing the JSON wrapper endpoint, if it exists or the original Callable.
+        """
+        return endpoint

pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]  # Specifies build tools to use
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "amq_adapter"
+version = "0.2.33"
+description = "The Python implementation of the AMQ Adapter for CleverMicro"
+readme = "README.md"
+authors = [
+    { name = "Stanislav Hejny", email = "stanislav.hejny@cleverthis.com" }
+]
+license = { text = "Apache License 2.0" }
+urls = { Homepage = "https://git.cleverthis.com/clevermicro/amq-adapter-python" }
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+requires-python = ">=3.11"
+dependencies = [
+    "aio-pika",
+    "aiohttp",
+    "fastapi",
+    "opentelemetry-api",
+    "opentelemetry-sdk",
+    "opentelemetry-exporter-otlp",
+    "opentelemetry-exporter-prometheus",
+    "opentelemetry-instrumentation-pika",
+    "pika",
+    "psutil",
+    "python_multipart"
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["amqp*"]
+
+[tool.black]
+line-length = 100
+
+[tool.pytest.ini_options]
+asyncio_default_fixture_loop_scope = "module"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "pytest-asyncio",
+    "build",
+    "twine",
+    "black",
+    "isort",
+    "flake8",
+    "pre-commit",
+    "pylint"
+]

requirements.txt

@@ -1,8 +0,0 @@
-aiohttp~=3.11.11
-pika~=1.3.2
-flask
-opentelemetry-api
-opentelemetry-sdk
-opentelemetry-exporter-otlp
-opentelemetry-exporter-prometheus
-opentelemetry-instrumentation-flask

tests/adapter/test_amq_route_factory.py

@@ -0,0 +1,30 @@
+from unittest import TestCase
+
+from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
+from amqp.model.model import AMQRoute
+
+
+class TestAMQRouteFactory(TestCase):
+    def test_from_string(self):
+        _f = AMQRouteFactory()
+        _r: AMQRoute = _f.from_string(
+            "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+        )
+        self.assertEqual("", _r.exchange)
+        self.assertEqual("amq-clever-pybook", _r.queue)
+        self.assertEqual(5, _r.timeout)
+        _r2: AMQRoute = _f.from_string(
+            "amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#:5"
+        )
+        self.assertEqual(NULL_ROUTE, _r2)
+
+    def test_validate(self):
+        _f = AMQRouteFactory()
+        self.assertTrue(
+            _f.validate(
+                "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+            )
+        )
+        self.assertFalse(
+            _f.validate("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#")
+        )

tests/adapter/test_backpressure_handler.py

@@ -0,0 +1,512 @@
+#
+# NOTE:   need to install: pytest-asyncio
+#
+
+import asyncio
+import json
+from asyncio import AbstractEventLoop
+from threading import Thread
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from aio_pika import Exchange
+from aio_pika.abc import AbstractRobustChannel
+
+from amqp.adapter.backpressure_handler import (
+    BackpressureHandler,
+    RunningAverage,
+    ScaleRequestV1,
+)
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import ScalingRequestAlert
+
+
+class TestScaleRequestV1:
+    """
+    A class containing pytest unit tests for the ScaleRequestV1 class.
+    """
+
+    def test_scale_request_v1_initialization(self):
+        """
+        Test that the ScaleRequestV1 object is initialized correctly.
+        """
+        service_id = "test-service"
+        task_id = "test-task"
+        max_availability = 10
+        current_availability = 5
+        request_type = ScalingRequestAlert.UPDATE
+
+        scale_request = ScaleRequestV1(
+            service_id,
+            task_id,
+            max_availability,
+            current_availability,
+            request_type,
+        )
+
+        assert scale_request.version == 1
+        assert scale_request.serviceId == service_id
+        assert scale_request.taskId == task_id
+        assert scale_request.maxAvailability == max_availability
+        assert scale_request.currentAvailability == current_availability
+        assert scale_request.requestType == request_type
+        assert scale_request.thread is None
+
+    def test_scale_request_v1_to_json(self):
+        """
+        Test the to_json method of ScaleRequestV1.
+        """
+        service_id = "test-service"
+        task_id = "test-task"
+        max_availability = 10
+        current_availability = 5
+        request_type = ScalingRequestAlert.UPDATE
+
+        scale_request = ScaleRequestV1(
+            service_id,
+            task_id,
+            max_availability,
+            current_availability,
+            request_type,
+        )
+
+        json_output = scale_request.to_json()
+        expected_json = json.dumps(
+            {
+                "version": 1,
+                "serviceId": service_id,
+                "taskId": task_id,
+                "maxAvailability": max_availability,
+                "currentAvailability": current_availability,
+                "requestType": request_type.value,
+            },
+            indent=2,
+        )
+        assert json_output == expected_json
+
+    def test_scale_request_v1_to_json_different_alert_type(self):
+        """
+        Test the to_json method with a different ScalingRequestAlert type.
+        """
+        service_id = "test-service"
+        task_id = "test-task"
+        max_availability = 10
+        current_availability = 5
+        request_type = ScalingRequestAlert.OVERLOAD  # Change the alert type
+
+        scale_request = ScaleRequestV1(
+            service_id,
+            task_id,
+            max_availability,
+            current_availability,
+            request_type,
+        )
+
+        json_output = scale_request.to_json()
+        expected_json = json.dumps(
+            {
+                "version": 1,
+                "serviceId": service_id,
+                "taskId": task_id,
+                "maxAvailability": max_availability,
+                "currentAvailability": current_availability,
+                "requestType": request_type.value,  # Ensure the value is correctly serialized
+            },
+            indent=2,
+        )
+        assert json_output == expected_json
+
+
+class TestRunningAverage:
+    """
+    A class containing pytest unit tests for the RunningAverage class.
+    """
+
+    def test_running_average_initialization(self):
+        """
+        Test that the RunningAverage object is initialized correctly.
+        """
+        num_items = 5
+        ra = RunningAverage(num_items)
+        assert ra.buffer == [0] * num_items
+        assert ra.pointer == 0
+        assert ra.num_items == num_items
+        assert ra.is_filled is False
+
+    def test_running_average_add_one_value(self):
+        """
+        Test adding a single value to the RunningAverage.
+        """
+        ra = RunningAverage(3)
+        ra.add(10)
+        assert ra.buffer == [10, 0, 0]
+        assert ra.pointer == 1
+        assert ra.is_filled is False
+
+    def test_running_average_add_multiple_values_within_capacity(self):
+        """
+        Test adding multiple values within the capacity of the RunningAverage buffer.
+        """
+        ra = RunningAverage(3)
+        ra.add(5)
+        ra.add(10)
+        ra.add(15)
+        assert ra.buffer == [5, 10, 15]
+        assert ra.pointer == 0
+        assert ra.is_filled is True
+
+    def test_running_average_add_more_values_than_capacity(self):
+        """
+        Test adding more values than the capacity of the RunningAverage buffer, checking wrap-around.
+        """
+        ra = RunningAverage(3)
+        ra.add(1)
+        ra.add(2)
+        ra.add(3)
+        ra.add(4)  # Overwrite the first value
+        ra.add(5)  # Overwrite the second value
+        assert ra.buffer == [4, 5, 3]
+        assert ra.pointer == 2
+        assert ra.is_filled is True
+
+    def test_running_average_get_average_empty(self):
+        """
+        Test get_average when no values have been added.
+        """
+        ra = RunningAverage(3)
+        assert ra.get_average() == 0.0
+
+    def test_running_average_get_average_one_value(self):
+        """
+        Test get_average after adding one value.
+        """
+        ra = RunningAverage(3)
+        ra.add(10)
+        assert ra.get_average() == 10.0
+
+    def test_running_average_get_average_multiple_values_within_capacity(self):
+        """
+        Test get_average with multiple values added within the buffer's capacity.
+        """
+        ra = RunningAverage(3)
+        ra.add(1)
+        ra.add(2)
+        ra.add(3)
+        assert ra.get_average() == 2.0
+
+    def test_running_average_get_average_more_values_than_capacity(self):
+        """
+        Test get_average after adding more values than the buffer's capacity (check wrap-around).
+        """
+        ra = RunningAverage(3)
+        ra.add(1)
+        ra.add(2)
+        ra.add(3)
+        ra.add(4)
+        ra.add(5)
+        assert ra.get_average() == 4.0
+
+    def test_get_current_values_empty(self):
+        """Test get_current_values when no values have been added."""
+        ra = RunningAverage(3)
+        assert ra.get_current_values() == []
+
+    def test_get_current_values_one_value(self):
+        """Test get_current_values after adding one value."""
+        ra = RunningAverage(3)
+        ra.add(10)
+        assert ra.get_current_values() == [10]
+
+    def test_get_current_values_multiple_values_within_capacity(self):
+        """Test get_current_values with multiple values within capacity."""
+        ra = RunningAverage(3)
+        ra.add(1)
+        ra.add(2)
+        ra.add(3)
+        assert ra.get_current_values() == [1, 2, 3]
+
+    def test_get_current_values_more_values_than_capacity(self):
+        """Test get_current_values after adding more values than capacity (wrap-around)."""
+        ra = RunningAverage(3)
+        ra.add(1)
+        ra.add(2)
+        ra.add(3)
+        ra.add(4)
+        ra.add(5)
+        assert ra.get_current_values() == [3, 4, 5]
+
+    def test_get_current_values_wrap_around(self):
+        """Test get_current_values when the buffer has wrapped around."""
+        ra = RunningAverage(4)
+        ra.add(1)
+        ra.add(2)
+        ra.add(3)
+        ra.add(4)
+        ra.add(5)  # Overwrite 1
+        ra.add(6)  # Overwrite 2
+        assert ra.get_current_values() == [3, 4, 5, 6]
+
+
+# Test for BackpressureHandler
+class TestBackpressureHandler:
+    """
+    A class containing pytest unit tests for the BackpressureHandler class.
+    """
+
+    @pytest.fixture
+    def mock_channel(self):
+        """
+        Pytest fixture to create a mock AbstractRobustChannel.
+        """
+        return AsyncMock(spec=AbstractRobustChannel)
+
+    @pytest.fixture
+    def mock_loop(self):
+        """
+        Pytest fixture to create a mock AbstractEventLoop.
+        """
+        return MagicMock(spec=AbstractEventLoop)
+
+    @pytest.fixture
+    def mock_config(self):
+        """
+        Pytest fixture to create a mock AMQConfiguration.
+        """
+        return AMQConfiguration("")
+
+    def test_backpressure_handler_initialization(self, mock_channel, mock_loop, mock_config):
+        """
+        Test that the BackpressureHandler object is initialized correctly.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        assert handler.channel == mock_channel
+        assert handler.loop == mock_loop
+        assert handler.config == mock_config
+        assert handler.swarm_service_id == "clever-amqp-service"
+        assert handler.swarm_task_id == "python-adapter"
+        assert handler.parallel_workers == mock_config.backpressure.threshold_threads
+        assert handler.average_cpu_usage is None
+
+    def test_increase_parallel_executions(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the increase_parallel_executions method.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        initial_executions = handler.current_parallel_executions
+        handler.increase_parallel_executions()
+        assert handler.current_parallel_executions == initial_executions + 1
+
+    def test_decrease_parallel_executions(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the decrease_parallel_executions method.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.current_parallel_executions = 5
+        handler.decrease_parallel_executions()
+        assert handler.current_parallel_executions == 4
+
+        handler.current_parallel_executions = 0
+        handler.decrease_parallel_executions()
+        assert handler.current_parallel_executions == 0  # Should not go below 0
+
+    def test_update_parallel_executions(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the update_parallel_executions method.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.update_parallel_executions(100)
+        assert handler.current_parallel_executions == 100
+
+    def test_update_last_data_message_time(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the update_last_data_message_time method.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        with patch("time.time") as mock_time:
+            mock_time.return_value = 12345
+            handler.update_last_data_message_time()
+            assert handler.last_data_message_time == 12345
+
+    def test_start_backpressure_monitor(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the start_backpressure_monitor method.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.do_loop = 1  # allow only 1 execution of the loop and then exit the thread, else tests will never end
+        thread = handler.start_backpressure_monitor()
+        assert isinstance(thread, Thread)
+        assert handler.thread == thread
+
+    @pytest.mark.asyncio
+    async def test_check_overload_condition_no_overload(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the check_overload_condition method when there is no overload.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.current_parallel_executions = 1
+        handler.parallel_workers = 5
+        handler.last_backpressure_event = ScalingRequestAlert.UPDATE
+        with patch.object(handler, "handle_backpressure_overload_event") as mock_handle_overload:
+            await handler.check_overload_condition()
+            mock_handle_overload.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_check_overload_condition_overload(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the check_overload_condition method when there is an overload.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.current_parallel_executions = 9  # Assuming parallel_workers is 10
+        handler.last_backpressure_event = ScalingRequestAlert.UPDATE
+        with patch.object(handler, "handle_backpressure_overload_event") as mock_handle_overload:
+            with patch("time.time") as mock_time:
+                mock_time.return_value = 12345
+                await handler.check_overload_condition()
+                mock_handle_overload.assert_called_once()
+                assert handler.last_backpressure_event_time == 12345
+                assert handler.last_backpressure_event == ScalingRequestAlert.OVERLOAD
+
+    @pytest.mark.asyncio
+    async def test_check_overload_condition_already_overload(
+        self, mock_channel, mock_loop, mock_config
+    ):
+        """
+        Test the check_overload_condition method when the last event was already an overload.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.current_parallel_executions = 10
+        handler.last_backpressure_event = ScalingRequestAlert.OVERLOAD
+        with patch.object(handler, "handle_backpressure_overload_event") as mock_handle_overload:
+            await handler.check_overload_condition()
+            mock_handle_overload.assert_not_called()
+
+    # backpressure_monitor_loop is difficult to test directly without significant mocking
+    # and potentially long-running tests.  The following is a basic outline of how
+    # you might approach testing it, focusing on verifying that the correct methods
+    # are called under specific conditions.  This is NOT a complete, runnable test.
+    @pytest.mark.asyncio
+    async def test_backpressure_monitor_loop(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the backpressure_monitor_loop method (outline).
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.average_cpu_usage = RunningAverage(
+            100, initial_value=99
+        )  # Mock the RunningAverage in OVERLOAD state
+        handler.average_cpu_usage.is_filled = True  # Together with 0 values ensures OVERLOAD state
+        # Setup mocks for dependent methods
+        handler.handle_backpressure_overload_event = AsyncMock()
+        handler._handle_backpressure_idle_event = AsyncMock()
+        handler.publish_backpressure_request = AsyncMock()
+
+        # 1. Simulate an overload condition:
+        handler.last_backpressure_event_time = 0  # Force event trigger
+        handler.last_data_message_time = 0
+        handler.do_loop = 1
+        handler.config.backpressure.cpu_overload_duration = (
+            -1
+        )  # Force overload condition even for recent change
+        await handler._backpressure_monitor()
+        handler.handle_backpressure_overload_event.assert_called_once()
+
+        # 2. Simulate an idle condition
+        handler.average_cpu_usage = RunningAverage(
+            100, initial_value=0
+        )  # Mock the RunningAverage in IDLE state
+        handler.average_cpu_usage.is_filled = True  # Together with 0 values ensures IDLE state
+        handler.last_backpressure_event_time = 0
+        handler.last_data_message_time = 0
+        handler.do_loop = 1
+        handler.config.backpressure.idle_duration = (
+            -1
+        )  # Force idle condition even for recent change
+        await handler._backpressure_monitor()
+        handler._handle_backpressure_idle_event.assert_called_once()
+
+        # 3. Simulate neither idle nor overload, but time_window passed
+        handler.handle_backpressure_overload_event.call_count = 0
+        handler._handle_backpressure_idle_event.call_count = 0
+        handler.average_cpu_usage = RunningAverage(100)  # Mock the RunningAverage in pritine state
+        handler.last_backpressure_event_time = 0
+        handler.last_data_message_time = 0
+        handler.do_loop = 1
+        await handler._backpressure_monitor()
+        assert (
+            handler.publish_backpressure_request.call_count
+            + handler.handle_backpressure_overload_event.call_count
+            + handler._handle_backpressure_idle_event.call_count
+        ) > 0
+
+    @pytest.mark.asyncio
+    async def test_handle_backpressure_overload_event(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the handle_backpressure_overload_event method.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.publish_backpressure_request = AsyncMock()
+        with patch("time.time") as mock_time:
+            mock_time.return_value = 12345
+            await handler.handle_backpressure_overload_event()
+        assert handler.publish_backpressure_request.call_count == 1
+        assert handler.last_data_message_time == 12345
+
+    @pytest.mark.asyncio
+    async def test_handle_backpressure_idle_event(self, mock_channel, mock_loop, mock_config):
+        """
+        Test the _handle_backpressure_idle_event method.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.publish_backpressure_request = AsyncMock()
+        with patch("time.time") as mock_time:
+            mock_time.return_value = 12345
+            await handler._handle_backpressure_idle_event()
+        assert handler.publish_backpressure_request.call_count == 1
+        assert handler.last_data_message_time == 12345
+
+    @pytest.mark.asyncio
+    async def test_publish_backpressure_request_exchange_exists(
+        self, mock_channel, mock_loop, mock_config
+    ):
+        """
+        Test the publish_backpressure_request method when the exchange already exists.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.exchange = AsyncMock(spec=Exchange)
+        with patch("asyncio.run_coroutine_threadsafe") as mock_async_run:
+            mock_async_run.return_value = asyncio.Future()
+            mock_async_run.return_value.set_result(None)
+            await handler.publish_backpressure_request(
+                ScaleRequestV1("test_service", "test_task", 100, 50, ScalingRequestAlert.UPDATE)
+            )
+        assert mock_async_run.call_count == 1
+        # handler.exchange.publish.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_publish_backpressure_request_exchange_does_not_exist(
+        self, mock_channel, mock_loop, mock_config
+    ):
+        """
+        Test the publish_backpressure_request method when the exchange does not exist and needs to be created.
+        """
+        handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
+        handler.exchange = None  # Simulate exchange not existing
+        mock_exchange = AsyncMock(spec=Exchange)
+        mock_channel.get_exchange.return_value = mock_exchange
+        BackpressureHandler._callback_list = {}
+
+        with patch("asyncio.run_coroutine_threadsafe") as mock_async_run:
+            mock_async_run.return_value = asyncio.Future()
+            mock_async_run.return_value.set_result(
+                mock_exchange
+            )  # Make it return the mock exchange
+            await handler.publish_backpressure_request(
+                ScaleRequestV1("test_service", "test_task", 100, 50, ScalingRequestAlert.UPDATE)
+            )
+            assert mock_async_run.call_count == 2
+
+        assert "_wrap_rabbit_mq_api_init" in BackpressureHandler._callback_list
+        assert "_wrap_rabbit_mq_api" in BackpressureHandler._callback_list
+        # Since I need also to mockup the asyncio.run_coroutine_threadsafe, the following mockups are
+        # not called because mockup overrides that call with preset return value.
+        # mock_channel.get_exchange.assert_called_once_with(name="cleverthis.clevermicro.management")
+        # handler.exchange.publish.assert_called_once()

tests/adapter/test_cleverthis_service_adapter.py

@@ -0,0 +1,431 @@
+import asyncio
+import unittest
+from enum import Enum
+from typing import Any, List, Optional
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from aio_pika.abc import AbstractChannel, AbstractRobustConnection
+from aiohttp import ClientResponse
+from aiohttp.web_fileresponse import FileResponse
+from pydantic import BaseModel
+
+from amqp.adapter.cleverthis_service_adapter import (
+    AMQMessage,
+    CleverThisServiceAdapter,
+    _convert_file_response,
+    _create_reply_exchange_and_queue,
+    is_likely_json,
+)
+from amqp.adapter.file_handler import FileHandler
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import DataResponse
+from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
+
+
+class NestedModel(BaseModel):
+    id: int
+    name: str
+
+
+class ComplexModel(BaseModel):
+    nested: NestedModel
+    items: List[str]
+
+
+class TestEnum(Enum):
+    VALUE1 = "value1"
+    VALUE2 = "value2"
+
+
+class TestModel(BaseModel):
+    name: str
+    value: int
+
+
+class TestCleverThisServiceAdapter(unittest.IsolatedAsyncioTestCase):
+
+    async def test_create_reply_exchange_and_queue_success(self):
+        connection_mock = AsyncMock()
+        channel_mock = AsyncMock()
+        exchange_mock = AsyncMock()
+        queue_mock = AsyncMock()
+
+        connection_mock.channel.return_value = channel_mock
+        channel_mock.get_exchange.return_value = exchange_mock
+        channel_mock.declare_queue.return_value = queue_mock
+        queue_mock.name = "test_queue_name"
+
+        channel, exchange, queue_name = await _create_reply_exchange_and_queue(connection_mock)
+
+        assert channel == channel_mock
+        assert exchange == exchange_mock
+        assert queue_name == "test_queue_name"
+        channel_mock.get_exchange.assert_called_once_with(name=AMQ_REPLY_TO_EXCHANGE, ensure=True)
+        channel_mock.declare_queue.assert_called_once_with(exclusive=False)
+        queue_mock.bind.assert_called_once_with(
+            exchange=AMQ_REPLY_TO_EXCHANGE, routing_key="test_queue_name"
+        )
+
+    async def test_convert_file_response_successful(self):
+        connection_mock = AsyncMock(spec=AbstractRobustConnection)
+        channel_mock: AbstractChannel = AsyncMock(spec=AbstractChannel)
+        exchange_mock = AsyncMock()
+        queue_mock = AsyncMock()
+
+        connection_mock.channel = AsyncMock(return_value=channel_mock)
+        channel_mock.get_exchange.return_value = exchange_mock
+        channel_mock.declare_queue.return_value = queue_mock
+        channel_mock.close = AsyncMock()
+        queue_mock.name = "test_queue_name"
+
+        mock_file_handler = MagicMock(spec=FileHandler)
+        mock_file_handler.publish_file = AsyncMock(return_value=("test_stream", 12345))
+        mock_file_response = MagicMock(spec=FileResponse)
+        mock_file_response.path = "/test/path"
+        mock_file_response.filename = "test_file.txt"
+        mock_loop = asyncio.get_event_loop()
+
+        json_response = await _convert_file_response(
+            mock_file_response, connection_mock, mock_file_handler, mock_loop
+        )
+
+        assert '"filename": "test_file.txt"' in json_response
+        assert '"streamName": "test_stream"' in json_response
+        assert '"fileSize": 12345' in json_response
+
+    def test_is_likely_json(self):
+        assert is_likely_json("{ }") is True
+        assert is_likely_json("[ ]") is True
+        assert is_likely_json("{]") is False
+        assert is_likely_json("[}") is False
+        assert is_likely_json("something else") is False
+
+    def setUp(self):
+        # Mock AMQConfiguration
+        self.mock_amq_config = AMQConfiguration("./application.properties.local")
+
+        # Mock ClientSession
+        self.mock_session = AsyncMock()
+
+        # Create adapter instance
+        self.adapter = CleverThisServiceAdapter(
+            amq_configuration=self.mock_amq_config, routes=[], session=self.mock_session
+        )
+        self.adapter.service_host = "testhost"
+        self.adapter.authenticated_user_arg_name = "user"
+
+    def test_init(self):
+        self.assertEqual(self.adapter.service_host, "testhost")
+        self.assertEqual(self.adapter.service_port, 8080)
+        self.assertFalse(self.adapter.require_authenticated_user)
+
+    def test_get_content_type(self):
+        test_cases = [
+            ({"Content-Type": ["application/json"]}, "application/json"),
+            ({"content-type": ["text/xml"]}, "text/xml"),
+            ({"CONTENT-TYPE": ["text/plain; charset=utf-8"]}, "text/plain"),
+            ({}, "application/json"),
+            ({"Other-Header": ["value"]}, "application/json"),
+        ]
+
+        for headers, expected in test_cases:
+            with self.subTest(headers=headers):
+                result = self.adapter.get_content_type(headers)
+                self.assertEqual(result, expected)
+
+    async def test_get_current_user(self):
+        # Base implementation should return None
+        result = await self.adapter.get_current_user("test_token")
+        self.assertIsNone(result)
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_unauthenticated(self, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers.return_value = {}
+        mock_message.path.return_value = "/some/path"
+        mock_message.method.return_value = "GET"
+        mock_message.id.return_value = "123"
+        mock_message.trace_info.return_value = {}
+
+        # Test when authentication is required but not provided
+        self.adapter.require_authenticated_user = True
+        response = await self.adapter.on_message(mock_message)
+        self.assertEqual(response.response_code(), "401")
+        mock_get_user.assert_not_called()
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    @patch.object(CleverThisServiceAdapter, "get")
+    async def test_on_message_authenticated(self, mock_get: MagicMock, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers.return_value = {"Authorization": ["Bearer valid_token"]}
+        mock_message.path.return_value = "/some/path"
+        mock_message.method.return_value = "GET"
+        mock_message.id.return_value = "123"
+        mock_message.trace_info.return_value = {}
+
+        mock_user = MagicMock()
+        mock_get_user.return_value = mock_user
+
+        mock_response = MagicMock(spec=DataResponse)
+        mock_get.return_value = mock_response
+
+        self.adapter.require_authenticated_user = True
+        response = await self.adapter.on_message(mock_message)
+
+        self.assertEqual(response, mock_response)
+        mock_get_user.assert_called_once_with("valid_token")
+        mock_get.assert_called_once_with(mock_message, mock_user)
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_login_path(self, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers.return_value = {}
+        mock_message.path.return_value = "/login"
+        mock_message.method.return_value = "POST"
+        mock_message.id.return_value = "123"
+        mock_message.trace_info.return_value = {}
+
+        mock_response = MagicMock(spec=DataResponse)
+        with patch.object(self.adapter, "post", return_value=mock_response) as mock_post:
+            response = await self.adapter.on_message(mock_message)
+            self.assertEqual(response, mock_response)
+            mock_get_user.assert_not_called()
+            mock_post.assert_called_once()
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_all_methods(self, mock_get_user):
+        mock_user = MagicMock()
+        mock_get_user.return_value = mock_user
+
+        mock_response = MagicMock(spec=DataResponse)
+
+        methods = ["GET", "PUT", "POST", "DELETE", "HEAD"]
+        for method in methods:
+            with self.subTest(method=method):
+                mock_message = MagicMock(spec=AMQMessage)
+                mock_message.headers.return_value = {"Authorization": ["Bearer valid_token"]}
+                mock_message.path.return_value = "/some/path"
+                mock_message.method.return_value = method
+                mock_message.id.return_value = "123"
+                mock_message.trace_info.return_value = {}
+
+                with patch.object(
+                    self.adapter, method.lower(), return_value=mock_response
+                ) as mock_method:
+                    response = await self.adapter.on_message(mock_message)
+                    self.assertEqual(response, mock_response)
+                    mock_method.assert_called_once()
+
+    async def test_get_with_session(self):
+        mock_message: AMQMessage = MagicMock(spec=AMQMessage)
+        mock_message.path.return_value = "/api/test"
+        mock_message.headers.return_value = {"X-Test": "value"}
+        mock_message.trace_info.return_value = {"trace-id": "123"}
+        mock_message.id.return_value = "msg123"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 200
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "success"}')
+
+        self.mock_session.get.return_value = mock_response
+
+        response = await self.adapter.get(mock_message, None)
+
+        self.assertEqual(response.response_code(), "200")
+        self.mock_session.get.assert_called_once_with(
+            "http://testhost:8080/api/test",
+            headers={"X-Test": "value", "trace-id": "123"},
+        )
+
+    async def test_get_without_session(self):
+        self.adapter.session = None
+
+        mock_message: AMQMessage = MagicMock(spec=AMQMessage)
+        mock_message.path.return_value = "/api/test"
+        mock_message.id.return_value = "msg123"
+        mock_message.trace_info.return_value = {}
+
+        response = await self.adapter.get(mock_message, None)
+
+        self.assertEqual(response.response_code(), "404")
+
+    async def test_put_with_form_handling(self):
+        mock_message: AMQMessage = MagicMock(spec=AMQMessage)
+        mock_message.path.return_value = "/api/test"
+        mock_message.headers.return_value = {}
+        mock_message.trace_info.return_value = {}
+        mock_message.id.return_value = "msg123"
+        mock_message.body.return_value = b"test=value"
+
+        mock_response: DataResponse = MagicMock(spec=DataResponse)
+        mock_response.response_code.return_value = "200"
+        mock_response.content_type.return_value = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "success"}')
+
+        self.mock_session.put.return_value = mock_response
+
+        with patch.object(
+            self.adapter,
+            "_authorize_forward_request",
+            wraps=self.adapter._authorize_forward_request,
+        ) as mock_handler:
+            response = await self.adapter.put(mock_message, None)
+
+            self.assertEqual(response.response_code(), "200")
+            mock_handler.assert_called_once()
+
+    async def test_post_with_form_handling(self):
+        mock_message: AMQMessage = MagicMock(spec=AMQMessage)
+        mock_message.path.return_value = "/api/test"
+        mock_message.headers.return_value = {}
+        mock_message.trace_info.return_value = {}
+        mock_message.id.return_value = "msg123"
+        mock_message.body.return_value = b"test=value"
+
+        mock_response: DataResponse = MagicMock(spec=DataResponse)
+        mock_response.response_code.return_value = "201"
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "created"}')
+
+        self.mock_session.post.return_value = mock_response
+
+        with patch.object(
+            self.adapter,
+            "_authorize_forward_request",
+            wraps=self.adapter._authorize_forward_request,
+        ) as mock_handler:
+            response = await self.adapter.post(mock_message, None)
+            self.assertEqual(response.response_code(), "201")
+            mock_handler.assert_called_once()
+
+    async def test_head_not_implemented(self):
+        mock_message: AMQMessage = MagicMock(spec=AMQMessage)
+        mock_message.id = "msg123"
+
+        with self.assertRaises(NotImplementedError):
+            await self.adapter.head(mock_message, None)
+
+    def test_basic_type_conversion(self):
+        async def test_method(x: int, y: float, z: bool, text: str):
+            pass
+
+        args = {"x": "123", "y": "45.67", "z": "true", "text": "hello"}
+
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["x"], 123)
+        self.assertEqual(result["y"], 45.67)
+        self.assertEqual(result["z"], True)
+        self.assertEqual(result["text"], "hello")
+
+    def test_enum_conversion(self):
+        async def test_method(status: TestEnum):
+            pass
+
+        args = {"status": "value1"}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["status"], TestEnum.VALUE1)
+
+    def test_optional_parameter(self):
+        async def test_method(required: str, optional: Optional[str] = None):
+            pass
+
+        args = {"required": "test"}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["required"], "test")
+        self.assertIsNone(result["optional"])
+
+    def test_list_parameter(self):
+        async def test_method(items: List[str]):
+            pass
+
+        args = {"items": "item1,item2,item3"}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["items"], ["item1", "item2", "item3"])
+
+    def test_pydantic_model_parameter(self):
+        async def test_method(model: TestModel):
+            pass
+
+        args = {"model": '{"name": "test", "value": 42}'}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertIsInstance(result["model"], TestModel)
+        self.assertEqual(result["model"].name, "test")
+        self.assertEqual(result["model"].value, 42)
+
+    def test_authenticated_user_parameter(self):
+        async def test_method(user: Any):
+            pass
+
+        user_obj = MagicMock()
+        args = {"user": user_obj}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["user"], user_obj)
+
+    def test_missing_required_parameter(self):
+        async def test_method(required: str):
+            pass
+
+        args = {}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertNotIn("required", result)
+
+    def test_invalid_enum_value(self):
+        async def test_method(status: TestEnum):
+            pass
+
+        args = {"status": "invalid_value"}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["status"], "invalid_value")
+
+    def test_invalid_type_conversion(self):
+        async def test_method(x: int):
+            pass
+
+        args = {"x": "not_a_number"}
+        result = {}
+        try:
+            result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+        except Exception as e:
+            self.assertTrue(isinstance(e, TypeError))
+
+        self.assertEqual(len(result), 0)
+
+    def test_complex_nested_model(self):
+
+        async def test_method(model: ComplexModel):
+            pass
+
+        args = {"model": {"nested": {"id": 1, "name": "test"}, "items": ["item1", "item2"]}}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertIsInstance(result["model"], ComplexModel)
+        self.assertEqual(result["model"].nested.id, 1)
+        self.assertEqual(result["model"].nested.name, "test")
+        self.assertEqual(result["model"].items, ["item1", "item2"])
+
+    def test_default_factory_parameter(self):
+        class DefaultFactory:
+            @staticmethod
+            def default_factory():
+                return "default_value"
+
+        async def test_method(param: str = DefaultFactory()):
+            pass
+
+        args = {}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["param"], "default_value")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/adapter/test_data_message_factory.py

@@ -0,0 +1,123 @@
+from datetime import datetime, timezone
+from unittest import TestCase
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+
+
+class DataMessageFactoryTest(TestCase):
+    def test_from_bytes(self):
+        raw = "\002\027A64f3b5af4d00000a4000|m=POST|d=clever-pybook.localhost|p=/debug|h={Accept=[*/*],X-Forwarded-Host=[clever-pybook.localhost:8085],User-Agent=[curl/8.7.1],X-Forwarded-Proto=[http],X-Forwarded-For=[172.18.0.1],Host=[clever-pybook.localhost:8085],Accept-Encoding=[gzip],Content-Length=[1475],X-Forwarded-Port=[8085],X-Real-Ip=[172.18.0.1],X-Forwarded-Server=[c5945bef67d1],Content-Type=[multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v]}|t={traceparent=00-bed6eb7de0d8ccf4c80b145a4bfec357-8d1f3bafc471bf0f-01}|b=TEST_ME"
+        msg = DataMessageFactory.from_bytes(raw.encode("utf-8"))
+        self.assertIsNotNone(msg.id)
+        self.assertEqual(
+            msg.id().as_string(),
+            "64F3B5AF4D00000A4000".lower(),
+            "SnowflakeID incorrectly parsed.",
+        )
+        self.assertEqual(
+            msg.headers()["Content-Type"][0],
+            "multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v",
+        )
+
+    def test_generate_snowflake_id(self):
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        self.assertEqual("00001000", _id.as_string()[-8:], "Instance ID not set correctly")
+
+        _id = DataMessageFactory(2).generate_snowflake_id()
+        self.assertEqual("00002000", _id.as_string()[-8:], "Instance ID not set correctly")
+
+        _f = DataMessageFactory(127)
+        _i0 = _f.generate_snowflake_id()
+        _i1 = _f.generate_snowflake_id()
+        _i2 = _f.generate_snowflake_id()
+        _i3 = _f.generate_snowflake_id()
+        self.assertEqual(
+            "0007F000".lower(),
+            _i0.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F001".lower(),
+            _i1.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F002".lower(),
+            _i2.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F003".lower(),
+            _i3.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+
+    def test_create_request_message(self):
+        _f = DataMessageFactory(34)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        self.assertEqual("localhost", _req.domain(), "Domain not set correctly")
+        self.assertEqual("/test/me", _req.path(), "Context path not set correctly")
+        self.assertEqual("v1", _req.headers()["k1"][0], "Headers not set correctly")
+        self.assertEqual(b"Quick brown fox", _req.body(), "Body not correctly decoded")
+
+    def test_serialize(self):
+        _f = DataMessageFactory(34)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _ser = _f.serialize(_req)
+        self.assertEqual(65, _ser[2])
+        _deser = _f.from_bytes(_ser)
+        self.assertEqual(_req.id().as_string(), _deser.id().as_string())
+        self.assertEqual("localhost", _deser.domain())
+        self.assertEqual("/test/me", _deser.path())
+        self.assertEqual(b"Quick brown fox", _deser.body())
+        self.assertEqual("v2", _deser.headers()["k2"][0])
+
+    def test_amq_message_routing_key(self):
+        _f = DataMessageFactory(35)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _key = _f.amq_message_routing_key(_req)
+        self.assertEqual(
+            "localhost.test.me",
+            _key,
+            "Message routing key wrongly extracted from the message",
+        )
+
+    def test_get_timestamp_from_id(self):
+        _f = DataMessageFactory(35)
+        _timestamp = _f.get_timestamp_from_id(_f.generate_snowflake_id())
+        _x_timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
+        self.assertLessEqual(
+            _x_timestamp - _timestamp,
+            1,
+            "Timestamp in SnowflakeId differs too much from current time",
+        )
+
+    def test_to_string(self):
+        _f = DataMessageFactory(35)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _as_str = _f.to_string(_req)
+        self.assertGreater(_as_str.index("domain='localhost'"), 0, "Expected content not found")

tests/adapter/test_data_parser.py

@@ -0,0 +1,165 @@
+import json
+import unittest
+from unittest.mock import MagicMock, patch
+
+from amqp.adapter.data_parser import AMQDataParser, MultipartFormDataParser
+
+
+class DummyAMQMessage:
+    def __init__(self, headers=None, body=b"", extra_data=None):
+        self._headers = headers or {"Content-Type": ["application/json"]}
+        self._body = body
+        self.extra_data = extra_data if extra_data is not None else {}
+
+    def headers(self):
+        return self._headers
+
+    def body(self):
+        return self._body
+
+
+class TestAMQDataParser(unittest.TestCase):
+    def test_process_form_data(self):
+        cases = [
+            (
+                '{"form": {"foo": ["bar"]}, "files": {"file1": ["data"]}}',
+                {"foo": "bar", "file1": "data"},
+            ),
+            ({"form": {"a": ["1"]}, "files": {"b": ["2"]}}, {"a": "1", "b": "2"}),
+            ({"form": {}, "files": {}}, {}),
+        ]
+        for json_input, expected in cases:
+            result = AMQDataParser.process_form_data(json_input)
+            self.assertEqual(result, expected)
+
+    def test_parse_get_url(self):
+        url = "http://example.com/path?foo=1&bar=2&foo=3"
+        path, params = AMQDataParser.parse_get_url(url)
+        self.assertEqual(path, "/path")
+        self.assertTrue(params["foo"] == ["1", "3"] or params["foo"] == "1" or params["foo"] == "3")
+        self.assertEqual(params["bar"], "2")
+
+    def test_parse_request_body_json(self):
+        data = {"foo": "bar"}
+        msg = DummyAMQMessage(
+            headers={"Content-Type": ["application/json"]}, body=json.dumps(data).encode()
+        )
+        result = AMQDataParser.parse_request_body(msg)
+        self.assertEqual(result, data)
+
+    def test_parse_request_body_invalid_json(self):
+        msg = DummyAMQMessage(headers={"Content-Type": ["application/json"]}, body=b"notjson")
+        with patch("amqp.adapter.data_parser.logging_error") as log_err:
+            with self.assertRaises(ValueError):
+                AMQDataParser.parse_request_body(msg)
+            self.assertTrue(log_err.called)
+
+    def test_parse_request_body_urlencoded(self):
+        msg = DummyAMQMessage(
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]}, body=b"foo=bar&baz=qux"
+        )
+        result = AMQDataParser.parse_request_body(msg)
+        self.assertEqual(result, {b"foo": b"bar", b"baz": b"qux"})
+
+    def test_parse_request_body_urlencoded_json(self):
+        data = {"form": {"foo": ["bar"]}, "files": {}}
+        msg = DummyAMQMessage(
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+            body=json.dumps(data).encode(),
+        )
+        result = AMQDataParser.parse_request_body(msg)
+        self.assertEqual(result, {"foo": "bar"})
+
+    def test_parse_request_body_invalid_urlencoded(self):
+        msg = DummyAMQMessage(
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]}, body=b"not a json"
+        )
+        with patch("amqp.adapter.data_parser.logging_error") as log_err:
+            with self.assertRaises(ValueError):
+                AMQDataParser.parse_request_body(msg)
+            self.assertTrue(log_err.called)
+
+    def test_parse_request_body_text(self):
+        msg = DummyAMQMessage(headers={"Content-Type": ["text/plain"]}, body=b"hello world")
+        result = AMQDataParser.parse_request_body(msg)
+        self.assertEqual(result, {"text": b"hello world"})
+
+    def test_parse_request_body_xml(self):
+        xml = b"<root><foo>bar</foo><baz>qux</baz></root>"
+        msg = DummyAMQMessage(headers={"Content-Type": ["application/xml"]}, body=xml)
+        result = AMQDataParser.parse_request_body(msg)
+        self.assertEqual(result, {"foo": "bar", "baz": "qux"})
+
+    def test_parse_request_body_invalid_xml(self):
+        msg = DummyAMQMessage(
+            headers={"Content-Type": ["application/xml"]}, body=b"<root><foo></bar></root>"
+        )
+        with self.assertRaises(ValueError):
+            AMQDataParser.parse_request_body(msg)
+
+    def test_parse_request_body_unsupported(self):
+        msg = DummyAMQMessage(headers={"Content-Type": ["application/unknown"]}, body=b"")
+        with self.assertRaises(ValueError):
+            AMQDataParser.parse_request_body(msg)
+
+
+class TestMultipartFormDataParser(unittest.TestCase):
+    def test_multipart_form_data_parser_json(self):
+        data = {"form": {"foo": ["bar"]}, "files": {}}
+        msg = DummyAMQMessage(
+            headers={"Content-Type": ["multipart/form-data"]},
+            body=json.dumps(data).encode(),
+            extra_data={"some": "data"},
+        )
+        parser = MultipartFormDataParser(msg)
+        self.assertTrue(parser.is_json)
+        self.assertEqual(parser.form_data, {"foo": "bar"})
+        self.assertTrue(parser.is_valid)
+
+    def test_multipart_form_data_parser_invalid_json(self):
+        msg = DummyAMQMessage(
+            headers={"Content-Type": ["multipart/form-data"]},
+            body=b"notjson",
+            extra_data={"some": "data"},
+        )
+        with patch("amqp.adapter.data_parser.logging_error") as log_err:
+            parser = MultipartFormDataParser(msg)
+            self.assertFalse(parser.is_valid)
+            self.assertTrue(log_err.called)
+
+    def test_multipart_form_data_parser_fallback_to_json(self):
+        msg = DummyAMQMessage(
+            headers={"Content-Type": ["multipart/form-data"]},
+            body=json.dumps({"form": {"foo": ["bar"]}, "files": {}}).encode(),
+        )
+        with patch(
+            "amqp.adapter.data_parser.python_multipart.parse_form", side_effect=Exception("fail")
+        ):
+            parser = MultipartFormDataParser(msg)
+            self.assertTrue(parser.is_json)
+            self.assertEqual(parser.form_data, {"foo": "bar"})
+            self.assertTrue(parser.is_valid)
+
+    def test_multipart_form_data_parser_on_field_and_file(self):
+        msg = DummyAMQMessage(
+            headers={"Content-Type": ["multipart/form-data"]}, body=b"", extra_data={}
+        )
+        parser = MultipartFormDataParser(msg)
+
+        class DummyField:
+            field_name = b"foo"
+            value = b"bar"
+
+        class DummyFile:
+            field_name = b"file"
+            file_object = MagicMock()
+            file_object.seek = MagicMock()
+            size = 123
+            file_name = b"filename.txt"
+
+        parser.on_field(DummyField())
+        self.assertEqual(parser.form_data["foo"], b"bar")
+        parser.on_file(DummyFile())
+        self.assertIn("file", parser.form_data)
+        self.assertTrue(hasattr(parser.form_data["file"], "filename"))
+        self.assertEqual(parser.form_data["file"].filename, "filename.txt")

tests/adapter/test_data_response_factory.py

@@ -0,0 +1,41 @@
+from unittest import TestCase
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.data_response_factory import DataResponseFactory
+from amqp.model.model import DataMessageMagicByte
+from amqp.model.snowflake_id import SnowflakeId
+
+
+class TestDataResponseFactory(TestCase):
+
+    def test_create_async_response_message(self):
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
+        _resp = _f.create_async_response_message(_g.generate_snowflake_id())
+        self.assertEqual("200", _resp.response_code())
+        self.assertEqual(b"OK", _resp.body())
+
+    def test_serialize(self):
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
+        _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
+        _resp = _f.create_async_response_message(_snowflakeId)
+        _ser = _f.serialize(_resp)
+        self.assertEqual(
+            2,
+            str(_ser[2:]).index(DataMessageMagicByte.HTTP_REQUEST.value + _snowflakeId.as_string()),
+            "Magic byte and SnowflakeId not found",
+        )
+        self.assertGreater(str(_ser[2:]).index("|e=|"), 2)
+        self.assertEqual((_ser[0] + _ser[1]), len(_ser) - 2 - len("T0s="))
+
+    def test_from_bytes(self):
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
+        _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
+        _resp = _f.create_async_response_message(_snowflakeId)
+        _ser = _f.serialize(_resp)
+        _deser = _f.from_bytes(_ser)
+        self.assertEqual(_snowflakeId.as_string(), _deser.id())
+        self.assertEqual("200", _deser.response_code())
+        self.assertEqual(b"OK", _deser.body())

tests/adapter/test_file_handler.py

@@ -0,0 +1,311 @@
+import asyncio
+import os
+import tempfile
+from asyncio import AbstractEventLoop
+from collections import defaultdict
+from pathlib import Path
+from shutil import rmtree
+from threading import Thread
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, MagicMock, mock_open, patch
+
+from aio_pika import IncomingMessage
+from aio_pika.abc import AbstractRobustConnection
+
+from amqp.adapter import file_handler
+from amqp.adapter.file_handler import StreamingFileHandler
+
+
+class TestFileHandler(IsolatedAsyncioTestCase):
+    def test_ensure_directory_exists(self):
+        """
+        Test function for ensure_directory_exists.  Creates a temporary directory
+        and files to test the versioning logic.
+        """
+
+        # Create a temporary directory
+        file_handler = StreamingFileHandler()
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Test case 1: File does not exist
+            filepath1 = os.path.join(temp_dir, "test1.txt")
+            result1 = file_handler.ensure_directory_exists(filepath1)
+            assert result1 == filepath1, f"Test Case 1 Failed: {result1} != {filepath1}"
+            # Create the file
+            with open(filepath1, "w") as f:
+                f.write("test")
+
+            # Test case 2: File exists, should create versioned file
+            filepath2 = os.path.join(temp_dir, "test2.txt")
+            result2 = file_handler.ensure_directory_exists(filepath2)
+            assert result2 == filepath2, f"Test Case 2 Failed: {result2} != {filepath2}"
+            with open(filepath2, "w") as f:
+                f.write("test")
+            result2_v1 = file_handler.ensure_directory_exists(filepath2)
+            assert result2_v1 == os.path.join(
+                temp_dir, "test2.1.txt"
+            ), f"Test Case 2 Version 1 Failed: {result2_v1} != {os.path.join(temp_dir, 'test2.1.txt')}"
+
+            # Create a few versioned files
+            with open(os.path.join(temp_dir, "test3.txt"), "w") as f:
+                f.write("test")
+            with open(os.path.join(temp_dir, "test3.1.txt"), "w") as f:
+                f.write("test")
+            with open(os.path.join(temp_dir, "test3.2.txt"), "w") as f:
+                f.write("test")
+            filepath3 = os.path.join(temp_dir, "test3.txt")
+            result3_v3 = file_handler.ensure_directory_exists(filepath3)
+            assert result3_v3 == os.path.join(
+                temp_dir, "test3.3.txt"
+            ), f"Test Case 3 Version 3 Failed: {result3_v3} != {os.path.join(temp_dir, 'test3.3.txt')}"
+
+            print("All test cases passed!")
+
+    async def test_download_from_queue_to_file(self):
+        loop = asyncio.new_event_loop()
+        tempFile = tempfile.mktemp()
+        file_handler = StreamingFileHandler()
+        with patch.object(
+            StreamingFileHandler, "_get_connection", new_callable=AsyncMock
+        ) as mock_connect:
+            mock_connection: AbstractRobustConnection = MagicMock(spec=AbstractRobustConnection)
+            mock_connect.return_value = mock_connection
+            mock_connection.channel = AsyncMock()
+            mock_channel = mock_connection.channel.return_value
+            mock_channel.get_queue = AsyncMock()
+            mock_queue = mock_channel.get_queue.return_value
+            mock_queue.iterator = MagicMock()
+            mock_queue_iter = mock_queue.iterator().__aenter__.return_value
+            mock_message: IncomingMessage = MagicMock(spec=IncomingMessage)
+            mock_message.body = b"test_data"
+            mock_message.body_size = len(b"test_data")
+            mock_message.headers = {"eof": 1}
+            mock_queue_iter.__aiter__.return_value = [mock_message]
+
+            # Act
+            result = await file_handler._download_from_queue_to_file(
+                "test_rabbitmq",
+                "test_queue",
+                filepath=tempFile,
+                loop=loop,
+            )
+            # clean up
+            os.remove(tempFile)
+
+            # Assert
+            assert result == (len(b"test_data"), 1)
+            mock_connect.assert_called_once_with("test_rabbitmq", loop=loop)
+            mock_channel.get_queue.assert_called_once_with("test_queue", ensure=False)
+
+    async def test_download_file_success(self):
+        # Arrange
+        file_handler = StreamingFileHandler()
+        loop = asyncio.new_event_loop()
+        loop_thread = Thread(target=loop.run_forever, daemon=True)
+        loop_thread.start()
+        rabbitmq_url = "amqp://test.rabbitmq"
+        file_info = {
+            "filename": "test_file.txt",
+            "size": 1024,
+            "streamName": "test_stream",
+            "queue_name": "test_queue",
+            "key": "test_key",
+        }
+        message_data = {}
+        output_dir = tempfile.mkdtemp()
+
+        file_handler.ensure_directory_exists = MagicMock(return_value=f"{output_dir}/test_file.txt")
+        file_handler._download_from_queue_to_file = AsyncMock(return_value=(123, 1))
+
+        # Act
+        result = await file_handler.download_file(
+            loop, rabbitmq_url, file_info, message_data, output_dir
+        )
+
+        # clean up
+        loop.stop()
+        rmtree(output_dir, ignore_errors=True)
+
+        # Assert
+        assert result == (123, 1)
+        file_handler.ensure_directory_exists.assert_called_once_with(f"{output_dir}/test_file.txt")
+        file_handler._download_from_queue_to_file.assert_called_once_with(
+            "amqp://test.rabbitmq", "test_queue", f"{output_dir}/test_file.txt", loop
+        )
+
+    async def test_download_file_queue_not_found(self):
+        # Arrange
+        file_handler = StreamingFileHandler()
+        loop = MagicMock(spec=AbstractEventLoop)
+        rabbitmq_url = "amqp://test.rabbitmq"
+        file_info = {
+            "filename": "test_file.txt",
+            "size": 1024,
+            "streamName": "test_stream",
+            "queue_name": None,
+            "key": "test_key",
+        }
+        message_data = {}
+        output_dir = "test_output"
+
+        # Act
+        result = await file_handler.download_file(
+            loop, rabbitmq_url, file_info, message_data, output_dir
+        )
+
+        # Assert
+        assert result == (0, 2)
+
+    async def test_download_from_queue_to_buffer(self):
+        loop = asyncio.new_event_loop()
+        file_handler = StreamingFileHandler()
+        with patch.object(
+            StreamingFileHandler, "_get_connection", new_callable=AsyncMock
+        ) as mock_connect:
+            mock_connection: AbstractRobustConnection = MagicMock(spec=AbstractRobustConnection)
+            mock_connect.return_value = mock_connection
+            mock_connection.channel = AsyncMock()
+            mock_channel = mock_connection.channel.return_value
+            mock_channel.get_queue = AsyncMock()
+            mock_queue = mock_channel.get_queue.return_value
+            mock_queue.iterator = MagicMock()
+            mock_queue_iter = mock_queue.iterator().__aenter__.return_value
+            mock_message: IncomingMessage = MagicMock(spec=IncomingMessage)
+            mock_message.body = b"test_data"
+            mock_message.body_size = len(b"test_data")
+            mock_message.headers = {"eof": 1}
+            mock_queue_iter.__aiter__.return_value = [mock_message]
+
+            # Act
+            result = await file_handler._download_from_queue_to_buffer(
+                "test_rabbitmq",
+                "test_queue",
+                loop=loop,
+            )
+
+            # Assert
+            assert result == (b"test_data", 1)
+            mock_connect.assert_called_once_with("test_rabbitmq", loop=loop)
+            mock_channel.get_queue.assert_called_once_with("test_queue", ensure=False)
+
+    async def test_download_buffer_success(self):
+        # Arrange
+        file_handler = StreamingFileHandler()
+        loop = asyncio.new_event_loop()
+        loop_thread = Thread(target=loop.run_forever, daemon=True)
+        loop_thread.start()
+        rabbitmq_url = "amqp://test.rabbitmq"
+
+        file_handler._download_from_queue_to_buffer = AsyncMock(return_value=(b"hello", 1))
+
+        # Act
+        result = await file_handler.download_buffer(loop, rabbitmq_url, "test_queue")
+
+        # clean up
+        loop.stop()
+
+        # Assert
+        assert result == (b"hello", 1)
+        file_handler._download_from_queue_to_buffer.assert_called_once_with(
+            "amqp://test.rabbitmq", "test_queue", loop
+        )
+
+    async def test_download_buffer_queue_not_found(self):
+        # Arrange
+        file_handler = StreamingFileHandler()
+        loop = MagicMock(spec=AbstractEventLoop)
+        rabbitmq_url = "amqp://test.rabbitmq"
+
+        # Act
+        result = await file_handler.download_buffer(loop, rabbitmq_url, None)
+
+        # Assert
+        assert result == (0, 2)
+
+    async def test_on_message_no_files_to_download(self):
+        handler = StreamingFileHandler()
+        message = MagicMock()
+        message.body = b"{}"
+        json_data = b'{"files": {}}'
+        rabbitmq_url = "amqp://localhost"
+        loop = AsyncMock()
+        output_dir = "downloaded_files"
+
+        handler.download_file = AsyncMock()
+        handler.download_file.return_value = (123, 1)
+        with patch("amqp.adapter.file_handler.logging_info"):
+            result = await handler.on_message(message, json_data, rabbitmq_url, loop, output_dir)
+
+        assert isinstance(result, defaultdict)
+        self.assertEqual(result, defaultdict())
+
+    async def test_on_message_with_files_to_download(self):
+        handler = StreamingFileHandler()
+        message = MagicMock()
+        message.body = b'{"fileStream1": "queue1"}'
+        json_data = b'{"files": {"key1": [{"streamName": "fileStream1"}]}}'
+        rabbitmq_url = "amqp://localhost"
+        loop = AsyncMock()
+        output_dir = "downloaded_files"
+
+        handler.download_file = AsyncMock()
+
+        with patch("amqp.adapter.file_handler.logging_info") as mock_logging_info:
+            result = await handler.on_message(message, json_data, rabbitmq_url, loop, output_dir)
+
+        self.assertEqual(result, file_handler.download_locations)
+        handler.download_file.assert_awaited_once()
+        mock_logging_info.assert_any_call(
+            f" [{message.delivery_tag}] about to create task for {{'streamName': 'fileStream1', 'queue_name': 'queue1', 'key': 'key1'}}"
+        )
+        mock_logging_info.assert_called_with(f"All files downloaded. d-tag: {message.delivery_tag}")
+
+    async def test_on_message_handles_exception(self):
+        handler = StreamingFileHandler()
+        message = MagicMock()
+        message.body = b"invalid_json"
+        message.ack = AsyncMock()
+        json_data = b"invalid_json"
+        rabbitmq_url = "amqp://localhost"
+        loop = AsyncMock()
+        output_dir = "downloaded_files"
+
+        with patch("amqp.adapter.file_handler.logging_error") as mock_logging_error:
+            await handler.on_message(message, json_data, rabbitmq_url, loop, output_dir)
+
+        mock_logging_error.assert_called()
+
+    async def test_publish_file_file_not_found(self):
+        file_handler = StreamingFileHandler()
+        loop = asyncio.get_event_loop()
+        exchange_mock = AsyncMock()
+        file_path = Path("/path/to/nonexistent_file.txt")
+
+        with self.assertRaises(FileNotFoundError):
+            await file_handler.publish_file(
+                exchange=exchange_mock,
+                file_path=file_path,
+                routing_key="test_routing_key",
+                loop=loop,
+            )
+
+    async def test_publish_file_success(self):
+        file_handler = StreamingFileHandler()
+        loop = asyncio.get_event_loop()
+        exchange_mock = AsyncMock()
+        file_path = Path("/path/to/test_file.txt")
+        data = b"test-data"
+        with (
+            patch("amqp.adapter.file_handler.Path.is_file", return_value=True),
+            patch("amqp.adapter.file_handler.os.path.getsize", return_value=len(data)),
+            patch("builtins.open", new=mock_open(read_data=data)) as mock_file,
+        ):
+            result = await file_handler.publish_file(
+                exchange=exchange_mock,
+                file_path=file_path,
+                routing_key="test_routing_key",
+                loop=loop,
+                max_chunk_size=512,
+            )
+
+            mock_file.assert_called_with(file_path, "rb")
+            self.assertEqual(result, ("test_routing_key", len(data)))

tests/adapter/test_logging_utils.py

@@ -0,0 +1,138 @@
+import os
+from importlib.metadata import version
+from unittest.mock import patch
+
+from amqp.adapter.logging_utils import (
+    get_context_info,
+    logging_debug,
+    logging_error,
+    logging_info,
+    logging_warning,
+)
+
+__version__ = version("amq_adapter")  # Name must match pyproject.toml [project] name
+__verbose__ = os.environ.get("AMQ_VERBOSE_LOGGING", "0") == "1"
+
+
+class TestLoggingFunctions:
+    """
+    A class containing pytest unit tests for the logging functions.
+    """
+
+    @patch("logging.error")
+    def test_logging_error_with_exception(self, mock_logger):
+        """
+        Test logging_error function when called within an exception handler.
+        """
+        try:
+            raise ValueError("Test error")
+        except ValueError as ve:
+            logging_error("An error occurred: %s", ve)
+        mock_logger.assert_called_once()
+        # Assert that the traceback information is included in the log message
+        assert len(mock_logger.call_args_list) == 1
+        _val = mock_logger.call_args_list[0][0][0]
+        assert "[E] 'An error occurred: %s'" in _val
+        assert "test_logging_error_with_exception" in _val
+
+    @patch("logging.error")
+    def test_logging_error_without_exception(self, mock_logger):
+        """
+        Test logging_error function when called without an active exception.
+        """
+        logging_error("A regular error")
+        mock_logger.assert_called_once()
+        assert "A regular error" in mock_logger.call_args_list[0][0][0]
+
+    @patch("logging.warning")
+    def test_logging_warning_verbose_on(self, mock_logger):
+        """
+        Test logging_warning function when verbose logging is enabled.
+        """
+        with patch(
+            "amqp.adapter.logging_utils.__verbose__", True
+        ):  # Simulate verbose logging
+            logging_warning("A warning message")
+        mock_logger.assert_called_once()
+        assert "A warning message" in mock_logger.call_args_list[0][0][0]
+
+    @patch("logging.warning")
+    def test_logging_warning_verbose_off(self, mock_logger):
+        """
+        Test logging_warning function when verbose logging is disabled.
+        """
+        with patch(
+            "amqp.adapter.logging_utils.__verbose__", False
+        ):  # Simulate non-verbose logging
+            logging_warning("A warning message")
+        mock_logger.assert_called_once()
+        assert "A warning message" in mock_logger.call_args_list[0][0][0]
+
+    @patch("logging.info")
+    def test_logging_info_verbose_on(self, mock_logger):
+        """
+        Test logging_info function with verbose logging enabled.
+        """
+        with patch("amqp.adapter.logging_utils.__verbose__", True):
+            logging_info("An info message")
+        mock_logger.assert_called_once()
+        assert "An info message" in mock_logger.call_args_list[0][0][0]
+
+    @patch("logging.info")
+    def test_logging_info_verbose_off(self, mock_logger):
+        """
+        Test logging_info function with verbose logging disabled.
+        """
+        with patch("amqp.adapter.logging_utils.__verbose__", False):
+            logging_info("An info message")
+        mock_logger.assert_called_once()
+        assert "An info message" in mock_logger.call_args_list[0][0][0]
+
+    @patch("logging.debug")
+    def test_logging_debug(self, mock_logger):
+        """
+        Test logging_debug function.
+        """
+        logging_debug("A debug message")
+        mock_logger.assert_called_once()
+        assert "A debug message" in mock_logger.call_args_list[0][0][0]
+
+    def test_get_context_info_normal(self):
+        """
+        Test get_context_info function in a normal function call.
+        """
+
+        def dummy_function():
+            return get_context_info()
+
+        module_name, line_number, function_name = dummy_function()
+        assert (
+            "test_get_context_info_normal" in function_name
+        )  # changed from co_name to function name
+        assert (
+            "test_logging_utils.py" in module_name
+        )  # changed from None to test_unit.py
+        assert isinstance(line_number, int)
+
+    def test_get_context_info_no_frame(self):
+        """
+        Test get_context_info function when no frame is available.
+        """
+        # Simulate a situation where inspect.currentframe() returns None (which is hard to do directly)
+        with patch("inspect.currentframe") as mock_frame:
+            mock_frame.return_value = None
+            module_name, line_number, function_name = get_context_info()
+            assert module_name is None
+            assert line_number is None
+            assert function_name is None
+
+    def test_get_context_info_exception(self):
+        """
+        Test get_context_info function when an exception occurs.
+        """
+        with patch("inspect.currentframe") as mock_frame:
+            mock_frame.side_effect = Exception("Simulated exception")
+            module_name, line_number, function_name = get_context_info()
+            assert module_name is None
+            assert line_number is None
+            assert function_name is None

tests/adapter/test_pydantic_serializer.py

@@ -0,0 +1,226 @@
+import json
+from datetime import date, datetime
+from unittest import TestCase
+from uuid import UUID
+
+from pydantic import BaseModel
+
+from amqp.adapter.pydantic_serializer import (
+    deserialize_object,
+    parse_url_query,
+    python_type_to_json,
+    serialize_object,
+)
+
+
+class NestedModel(BaseModel):
+    """
+    Nested model for testing the pydantic serializer.
+    """
+
+    value: str
+    number: int
+
+
+class TestModel(BaseModel):
+    """
+    Test model for testing the pydantic serializer.
+    """
+
+    name: str
+    age: int
+    uuid_field: UUID
+    datetime_field: datetime
+    nested_model: NestedModel | None = None
+    list_field: list = []
+
+
+class PydanticSerializerTest(TestCase):
+
+    def setUp(self):
+        """
+        Prepare the test data.
+        """
+        self.test_uuid = UUID("12345678-1234-5678-1234-567812345678")
+        self.test_datetime = datetime(2024, 1, 1, 12, 0, 0)
+        self.nested_model = NestedModel(value="test", number=42)
+        self.test_model = TestModel(
+            name="test",
+            age=25,
+            uuid_field=self.test_uuid,
+            datetime_field=self.test_datetime,
+            nested_model=self.nested_model,
+            list_field=[1, 2, 3],
+        )
+
+    def test_serialize_object(self):
+        serialized = serialize_object(self.test_model)
+        self.assertIsInstance(serialized, str)
+        self.assertIn("TestModel", serialized)
+        self.assertIn("test", serialized)
+        self.assertIn("25", serialized)
+        self.assertIn(str(self.test_uuid), serialized)
+        self.assertIn(self.test_datetime.isoformat(), serialized)
+
+    def test_deserialize_object(self):
+        serialized = serialize_object(self.test_model)
+        deserialized = deserialize_object(serialized)
+
+        self.assertIsInstance(deserialized, TestModel)
+        self.assertEqual(deserialized.name, "test")
+        self.assertEqual(deserialized.age, 25)
+        self.assertEqual(deserialized.uuid_field, self.test_uuid)
+        self.assertEqual(deserialized.datetime_field, self.test_datetime)
+        self.assertIsInstance(deserialized.nested_model, NestedModel)
+        self.assertEqual(deserialized.nested_model.value, "test")
+        self.assertEqual(deserialized.nested_model.number, 42)
+        self.assertEqual(deserialized.list_field, [1, 2, 3])
+
+    def test_serialize_deserialize_with_none(self):
+        model = TestModel(
+            name="test",
+            age=25,
+            uuid_field=self.test_uuid,
+            datetime_field=self.test_datetime,
+            nested_model=None,
+            list_field=[],
+        )
+        serialized = serialize_object(model)
+        deserialized = deserialize_object(serialized)
+        self.assertIsNone(deserialized.nested_model)
+
+    def test_serialize_deserialize_with_list_of_models(self):
+        nested_models = [NestedModel(value=f"test{i}", number=i) for i in range(3)]
+        model = TestModel(
+            name="test",
+            age=25,
+            uuid_field=self.test_uuid,
+            datetime_field=self.test_datetime,
+            nested_model=None,
+            list_field=nested_models,
+        )
+        serialized = serialize_object(model)
+        deserialized = deserialize_object(serialized)
+        self.assertEqual(len(deserialized.list_field), 3)
+        for i, item in enumerate(deserialized.list_field):
+            self.assertIsInstance(item, NestedModel)
+            self.assertEqual(item.value, f"test{i}")
+            self.assertEqual(item.number, i)
+
+    def test_parse_url_query_no_params(self):
+        test_url = "http://example.com/path"
+        path, params = parse_url_query(test_url)
+
+        self.assertEqual(path, "/path")
+        self.assertEqual(params, {})
+
+    def test_parse_url_query_single_value(self):
+        test_url = "http://example.com/path?foo=1"
+        path, params = parse_url_query(test_url)
+
+        self.assertEqual(path, "/path")
+        self.assertEqual(params["foo"], "1")
+
+    def test_parse_url_query(self):
+        path, args = parse_url_query(
+            "http://localhost:8080/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741?response_type=JsonFile"
+        )
+        self.assertEqual(path, "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741")
+        self.assertEqual(args, {"response_type": "JsonFile"})
+        path, args = parse_url_query(
+            "http://localhost:8080/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        )
+        self.assertEqual(path, "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741")
+        self.assertEqual(args, {})
+
+
+class TestPythonTypeToJson:
+    """
+    Test cases for the python_type_to_json function.
+    """
+
+    def test_basic_types(self):
+        """Test serialization of basic Python types."""
+        # Test dictionary
+        data = {"key": "value", "number": 42}
+        result = python_type_to_json(data)
+        assert json.loads(result) == data
+
+        # Test list
+        data = [1, 2, 3, "four"]
+        result = python_type_to_json(data)
+        assert json.loads(result) == data
+
+        # Test tuple
+        data = (1, 2, 3)
+        result = python_type_to_json(data)
+        assert json.loads(result) == [1, 2, 3]  # Tuples are converted to lists
+
+        # Test set
+        data = {1, 2, 3}
+        result = python_type_to_json(data)
+        assert set(json.loads(result)) == data  # Sets are converted to lists
+
+    def test_datetime_handling(self):
+        """Test serialization of datetime objects."""
+        # Test datetime
+        test_datetime = datetime(2024, 1, 1, 12, 0, 0)
+        data = {"timestamp": test_datetime}
+        result = python_type_to_json(data)
+        assert json.loads(result) == {"timestamp": test_datetime.isoformat()}
+
+        # Test date
+        test_date = date(2024, 1, 1)
+        data = {"date": test_date}
+        result = python_type_to_json(data)
+        assert json.loads(result) == {"date": test_date.isoformat()}
+
+    def test_nested_structures(self):
+        """Test serialization of nested data structures."""
+        data = {
+            "list_of_datetimes": [datetime(2024, 1, 1), datetime(2024, 1, 2)],
+            "dict_with_date": {"date": date(2024, 1, 1)},
+            "mixed": {"set": {1, 2, 3}, "tuple": (1, 2, 3), "datetime": datetime(2024, 1, 1)},
+        }
+        result = python_type_to_json(data)
+        parsed = json.loads(result)
+        assert parsed["list_of_datetimes"] == [d.isoformat() for d in data["list_of_datetimes"]]
+        assert parsed["dict_with_date"]["date"] == data["dict_with_date"]["date"].isoformat()
+        assert set(parsed["mixed"]["set"]) == data["mixed"]["set"]
+        assert parsed["mixed"]["tuple"] == list(data["mixed"]["tuple"])
+        assert parsed["mixed"]["datetime"] == data["mixed"]["datetime"].isoformat()
+
+    def test_custom_object_serialization(self):
+        """Test serialization of custom objects with __dict__ attribute."""
+
+        class TestObject:
+            def __init__(self, value):
+                self.value = value
+                self.timestamp = datetime(2024, 1, 1)
+
+        obj = TestObject("test")
+        data = {"obj": obj}
+        result = python_type_to_json(data)
+        parsed = json.loads(result)
+        assert parsed["obj"]["value"] == "test"
+        assert parsed["obj"]["timestamp"] == obj.timestamp.isoformat()
+
+    def test_empty_structures(self):
+        """Test serialization of empty data structures."""
+        # Empty dict
+        assert python_type_to_json({}) == "{}"
+
+        # Empty list
+        assert python_type_to_json([]) == "[]"
+
+        # Empty set
+        assert python_type_to_json(set()) == "[]"
+
+        # Empty tuple
+        assert python_type_to_json(()) == "[]"
+
+    def test_unicode_handling(self):
+        """Test handling of Unicode characters."""
+        data = {"text": "Hello, 世界!"}
+        result = python_type_to_json(data)
+        assert json.loads(result) == data  # ensure_ascii=False should preserve Unicode

tests/adapter/test_serializer.py

@@ -0,0 +1,114 @@
+import unittest
+from io import BytesIO
+
+from amqp.adapter.serializer import (
+    bytes_to_long,
+    csv_as_list,
+    long_to_bytes,
+    map_as_string,
+    map_with_list_as_string,
+    object_or_list_as_string,
+    parse_map_list_string,
+    parse_map_string,
+    read_long,
+    str_to_bool,
+)
+
+
+class TestSerializer(unittest.TestCase):
+    def test_long_to_bytes_and_bytes_to_long(self):
+        # Test conversion in both directions
+        test_numbers = [
+            -9223372036854775808,
+            -1,
+            0,
+            1,
+            9223372036854775807,
+        ]  # Min and max long values
+        for num in test_numbers:
+            bytes_val = long_to_bytes(num)
+            self.assertEqual(bytes_to_long(bytes_val), num)
+
+    def test_read_long(self):
+        # Test reading valid long
+        test_num = 12345
+        bio = BytesIO(long_to_bytes(test_num))
+        self.assertEqual(read_long(bio), test_num)
+
+        # Test reading from empty BytesIO
+        empty_bio = BytesIO()
+        self.assertEqual(read_long(empty_bio), 0)
+
+    def test_object_or_list_as_string(self):
+        # Test with string
+        self.assertEqual(object_or_list_as_string("test"), "test")
+
+        # Test with list
+        self.assertEqual(object_or_list_as_string(["a", "b", "c"]), "[a,b,c]")
+
+        # Test with empty list
+        self.assertEqual(object_or_list_as_string([]), "[]")
+
+    def test_map_as_string(self):
+        # Test with simple key-value pairs
+        test_map = {"key1": "value1", "key2": "value2"}
+        self.assertEqual(map_as_string(test_map), "key1=value1,key2=value2")
+
+        # Test with empty map
+        self.assertEqual(map_as_string({}), "")
+
+    def test_map_with_list_as_string(self):
+        # Test with lists as values
+        test_map = {"key1": ["a", "b"], "key2": ["c", "d"]}
+        self.assertEqual(map_with_list_as_string(test_map), "key1=[a,b],key2=[c,d]")
+
+        # Test with empty map
+        self.assertEqual(map_with_list_as_string({}), "")
+
+    def test_parse_map_string(self):
+        # Test valid map string
+        test_str = "{key1=value1,key2=value2}"
+        expected = {"key1": "value1", "key2": "value2"}
+        self.assertEqual(parse_map_string(test_str), expected)
+
+        # Test empty map string
+        self.assertEqual(parse_map_string("{}"), {})
+
+        # Test invalid format
+        self.assertEqual(parse_map_string("invalid"), {})
+
+    def test_csv_as_list(self):
+        # Test normal CSV string
+        self.assertEqual(csv_as_list("a,b,c"), ["a", "b", "c"])
+
+        # Test with spaces
+        self.assertEqual(csv_as_list(" a , b , c "), ["a", "b", "c"])
+
+        # Test with brackets
+        self.assertEqual(csv_as_list("[a,b,c]"), ["a", "b", "c"])
+
+        # Test empty list
+        self.assertEqual(csv_as_list("[]"), [""])
+
+    def test_parse_map_list_string(self):
+        # Test valid map list string
+        test_str = "{key1=[a,b],key2=[c,d]}"
+        expected = {"key1": ["a", "b"], "key2": ["c", "d"]}
+        self.assertEqual(parse_map_list_string(test_str), expected)
+
+        # Test empty map
+        self.assertEqual(parse_map_list_string("{}"), {})
+
+        # Test single entry
+        self.assertEqual(parse_map_list_string("{key1=[a,b]}"), {"key1": ["a", "b"]})
+
+    def test_str_to_bool(self):
+        # Test true values
+        true_values = ["true", "yes", "1", "t", "y", "on", "TRUE", "YES", "T", "Y", "ON"]
+        for value in true_values:
+            self.assertTrue(str_to_bool(value))
+
+        # Test false values
+        false_values = ["false", "no", "0", "f", "n", "off", "FALSE", "NO", "F", "N", "OFF"]
+        for value in false_values:
+            self.assertFalse(str_to_bool(value))

tests/adapter/test_service_message_factory.py

@@ -1,25 +1,32 @@
 from unittest import TestCase
 
-from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory, RS
-from amqp.model.model import ServiceMessageType, CleverMicroServiceMessage
+from amqp.adapter.service_message_factory import (
+    INVALID_MESSAGE,
+    RS,
+    ServiceMessageFactory,
+)
+from amqp.model.model import ServiceMessage
+from amqp.model.service_message_type import ServiceMessageType
 
 
-class CleverMicroServiceMessageTest(TestCase):
+class ServiceMessageTest(TestCase):
 
     def test_to_string(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory('bumble-bee')
-        message: CleverMicroServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
+        _f: ServiceMessageFactory = ServiceMessageFactory("bumble-bee")
+        message: ServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         message_str = _f.to_string(message)
-        assert "|type=01|" in message_str
-        assert "|body=Duck" in message_str
-        message: CleverMicroServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald")
+        self.assertTrue("|type=01|" in message_str)
+        self.assertTrue("|body=Duck" in message_str)
+        message: ServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald")
         message_str = _f.to_string(message)
-        assert "|type=03|" in message_str
-        assert "|body=Duck" in message_str
-        assert "|replyTo=bumble-bee" in message_str
+        self.assertTrue("|type=03|" in message_str)
+        self.assertTrue("|body=Duck" in message_str)
+        self.assertTrue("|replyTo=bumble-bee" in message_str)
+        nullMsg = _f.to_string(None)
+        self.assertEqual("null", nullMsg)
 
     def test_from(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory("amqp-router")
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
         message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         deserialized = _f.from_bytes(serialized)
@@ -28,9 +35,11 @@ class CleverMicroServiceMessageTest(TestCase):
         assert deserialized.recipient_name == "Donald"
         assert deserialized.message == "Duck"
         assert deserialized.reply_to == "amqp-router"
+        invalidMsg = _f.from_bytes(b"wrong")
+        self.assertEqual(invalidMsg, INVALID_MESSAGE)
 
     def test_base64(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory("amqp-router")
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
         message = _f.as_base64(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         assert b"|81|" in serialized
@@ -38,9 +47,10 @@ class CleverMicroServiceMessageTest(TestCase):
         assert deserialized.is_valid()
         assert deserialized.message_type == ServiceMessageType.ROUTING_DATA
         assert deserialized.message == "Duck"
-    
+
     def test_serialize(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory("amqp-router")
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
         message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         assert RS + "0" + str(ServiceMessageType.ROUTING_DATA.value)
+        assert len(serialized) > 16

tests/adapter/test_trace_info_adapter.py

@@ -0,0 +1,251 @@
+"""Unit tests for the tracing module."""
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.trace import Tracer
+
+from amqp.service.tracing import (
+    MetricsConfig,
+    TracingConfig,
+    create_resource,
+    create_span,
+    initialize_telemetry,
+    setup_metrics_provider,
+    setup_otlp_exporter,
+    setup_trace_provider,
+)
+
+
+class TestTracingConfig(unittest.TestCase):
+    """Test cases for TracingConfig."""
+
+    def test_default_values(self):
+        """Test that default values are set correctly."""
+        config = TracingConfig(service_name="test-service", service_version="1.0.0")
+        self.assertEqual(config.service_name, "test-service")
+        self.assertEqual(config.service_version, "1.0.0")
+        self.assertEqual(config.otlp_endpoint, "http://localhost:4317")
+        self.assertTrue(config.insecure)
+        self.assertFalse(config.debug)
+        self.assertIsNone(config.additional_attributes)
+
+    def test_custom_values(self):
+        """Test that custom values are set correctly."""
+        config = TracingConfig(
+            service_name="test-service",
+            service_version="1.0.0",
+            otlp_endpoint="http://custom:4317",
+            insecure=False,
+            debug=True,
+            additional_attributes={"custom": "attribute"},
+        )
+        self.assertEqual(config.otlp_endpoint, "http://custom:4317")
+        self.assertFalse(config.insecure)
+        self.assertTrue(config.debug)
+        self.assertEqual(config.additional_attributes, {"custom": "attribute"})
+
+
+class TestMetricsConfig(unittest.TestCase):
+    """Test cases for MetricsConfig."""
+
+    def test_default_values(self):
+        """Test that default values are set correctly."""
+        config = MetricsConfig()
+        self.assertEqual(config.prometheus_port, 9464)
+        self.assertEqual(config.prometheus_host, "localhost")
+        self.assertEqual(config.export_interval_millis, 30000)
+        self.assertIsNone(config.additional_attributes)
+
+    def test_custom_values(self):
+        """Test that custom values are set correctly."""
+        config = MetricsConfig(
+            prometheus_port=8000,
+            prometheus_host="0.0.0.0",
+            export_interval_millis=15000,
+            additional_attributes={"custom": "attribute"},
+        )
+        self.assertEqual(config.prometheus_port, 8000)
+        self.assertEqual(config.prometheus_host, "0.0.0.0")
+        self.assertEqual(config.export_interval_millis, 15000)
+        self.assertEqual(config.additional_attributes, {"custom": "attribute"})
+
+
+class TestCreateResource(unittest.TestCase):
+    """Test cases for create_resource function."""
+
+    def test_create_resource_basic(self):
+        """Test creating a resource with basic configuration."""
+        config = TracingConfig(service_name="test-service", service_version="1.0.0")
+        resource = create_resource(config)
+        self.assertIsInstance(resource, Resource)
+        self.assertEqual(resource.attributes.get("service.name"), "test-service")
+        self.assertEqual(resource.attributes.get("service.version"), "1.0.0")
+
+    def test_create_resource_with_attributes(self):
+        """Test creating a resource with additional attributes."""
+        config = TracingConfig(
+            service_name="test-service",
+            service_version="1.0.0",
+            additional_attributes={"custom": "attribute"},
+        )
+        resource = create_resource(config)
+        self.assertEqual(resource.attributes.get("custom"), "attribute")
+
+
+class TestSetupExporters(unittest.TestCase):
+    """Test cases for exporter setup functions."""
+
+    @patch("amqp.service.tracing.OTLPSpanExporter")
+    def test_setup_otlp_exporter(self, mock_otlp):
+        """Test OTLP exporter setup."""
+        config = TracingConfig(
+            service_name="test-service",
+            service_version="1.0.0",
+            otlp_endpoint="http://jaeger:4317",
+            insecure=True,
+        )
+        setup_otlp_exporter(config)
+        mock_otlp.assert_called_once_with(endpoint="http://jaeger:4317", insecure=True)
+
+    @patch("amqp.service.tracing.OTLPSpanExporter")
+    def test_setup_otlp_exporter_secure(self, mock_otlp):
+        """Test OTLP exporter setup with secure connection."""
+        config = TracingConfig(
+            service_name="test-service",
+            service_version="1.0.0",
+            otlp_endpoint="https://jaeger:4317",
+            insecure=False,
+        )
+        setup_otlp_exporter(config)
+        mock_otlp.assert_called_once_with(endpoint="https://jaeger:4317", insecure=False)
+
+
+class TestSetupTraceProvider(unittest.TestCase):
+    """Test cases for trace provider setup."""
+
+    @patch("amqp.service.tracing.TracerProvider")
+    @patch("amqp.service.tracing.BatchSpanProcessor")
+    @patch("amqp.service.tracing.setup_otlp_exporter")
+    def test_setup_trace_provider_basic(self, mock_otlp_exporter, mock_processor, mock_provider):
+        """Test basic trace provider setup."""
+        config = TracingConfig(service_name="test-service", service_version="1.0.0")
+        resource = create_resource(config)
+
+        mock_provider_instance = MagicMock()
+        mock_provider.return_value = mock_provider_instance
+
+        setup_trace_provider(config, resource)
+
+        mock_provider.assert_called_once_with(resource=resource)
+        mock_otlp_exporter.assert_called_once()
+        self.assertEqual(mock_provider_instance.add_span_processor.call_count, 1)
+
+    @patch("amqp.service.tracing.TracerProvider")
+    @patch("amqp.service.tracing.BatchSpanProcessor")
+    @patch("amqp.service.tracing.setup_otlp_exporter")
+    @patch("amqp.service.tracing.ConsoleSpanExporter")
+    def test_setup_trace_provider_with_debug(
+        self, mock_console_exporter, mock_otlp_exporter, mock_processor, mock_provider
+    ):
+        """Test trace provider setup with debug mode."""
+        config = TracingConfig(service_name="test-service", service_version="1.0.0", debug=True)
+        resource = create_resource(config)
+
+        mock_provider_instance = MagicMock()
+        mock_provider.return_value = mock_provider_instance
+
+        setup_trace_provider(config, resource)
+
+        self.assertEqual(mock_provider_instance.add_span_processor.call_count, 2)
+        mock_console_exporter.assert_called_once()
+
+
+class TestSetupMetricsProvider(unittest.TestCase):
+    """Test cases for metrics provider setup."""
+
+    @patch("amqp.service.tracing.start_http_server")
+    @patch("amqp.service.tracing.PrometheusMetricReader")
+    @patch("amqp.service.tracing.PeriodicExportingMetricReader")
+    @patch("amqp.service.tracing.MeterProvider")
+    def test_setup_metrics_provider_success(
+        self, mock_meter_provider, mock_periodic_reader, mock_prometheus_reader, mock_start_server
+    ):
+        """Test successful metrics provider setup."""
+        config = MetricsConfig(prometheus_port=9464, prometheus_host="localhost")
+        resource = Resource({})
+
+        setup_metrics_provider(config, resource)
+
+        mock_start_server.assert_called_once_with(port=9464, addr="localhost")
+        mock_prometheus_reader.assert_called_once()
+        mock_periodic_reader.assert_called_once()
+        mock_meter_provider.assert_called_once()
+
+    @patch("amqp.service.tracing.start_http_server")
+    def test_setup_metrics_provider_failure(self, mock_start_server):
+        """Test metrics provider setup with failure."""
+        mock_start_server.side_effect = Exception("Failed to start server")
+
+        config = MetricsConfig()
+        resource = Resource({})
+
+        provider = setup_metrics_provider(config, resource)
+        self.assertIsNotNone(provider)  # Should return no-op provider
+
+
+class TestInitializeTelemetry(unittest.TestCase):
+    """Test cases for initialize_telemetry function."""
+
+    @patch("amqp.service.tracing.setup_trace_provider")
+    @patch("amqp.service.tracing.setup_metrics_provider")
+    @patch("amqp.service.tracing.trace")
+    @patch("amqp.service.tracing.metrics")
+    def test_initialize_telemetry_success(
+        self, mock_metrics, mock_trace, mock_setup_metrics, mock_setup_trace
+    ):
+        """Test successful telemetry initialization."""
+        config = TracingConfig(service_name="test-service", service_version="1.0.0")
+        metrics_config = MetricsConfig()
+
+        mock_tracer = MagicMock(spec=Tracer)
+        mock_trace.get_tracer.return_value = mock_tracer
+
+        tracer = initialize_telemetry(config, metrics_config)
+
+        self.assertEqual(tracer, mock_tracer)
+        mock_setup_trace.assert_called_once()
+        mock_setup_metrics.assert_called_once()
+
+    @patch("amqp.service.tracing.setup_trace_provider")
+    def test_initialize_telemetry_failure(self, mock_setup_trace):
+        """Test telemetry initialization with failure."""
+        mock_setup_trace.side_effect = Exception("Setup failed")
+
+        tracer = initialize_telemetry()
+
+        self.assertIsNotNone(tracer)  # Should return no-op tracer
+
+
+class TestCreateSpan(unittest.TestCase):
+    """Test cases for create_span function."""
+
+    def test_create_span_basic(self):
+        """Test basic span creation."""
+        mock_tracer = MagicMock(spec=Tracer)
+        create_span(mock_tracer, "test-span")
+        mock_tracer.start_as_current_span.assert_called_once_with("test-span", attributes={})
+
+    def test_create_span_with_attributes(self):
+        """Test span creation with attributes."""
+        mock_tracer = MagicMock(spec=Tracer)
+        attributes = {"custom": "attribute"}
+        create_span(mock_tracer, "test-span", attributes)
+        mock_tracer.start_as_current_span.assert_called_once_with(
+            "test-span", attributes=attributes
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/adapter/test_type_descriptor.py

@@ -0,0 +1,211 @@
+import json
+import unittest
+
+from pydantic import BaseModel
+
+from amqp.adapter.type_descriptor import TypeDescriptor, _instantiate
+
+
+class TestClass(BaseModel):
+    value: str
+    extra: str
+
+
+class TestTypeDescriptor(unittest.TestCase):
+    def test_annotated_with_json(self):
+        type_str = (
+            "typing.Optional[typing.Annotated[core.base.providers.ingestion.IngestionConfig, Json]]"
+        )
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(td.type, "core.base.providers.ingestion.IngestionConfig")
+        self.assertTrue(td.is_json)
+
+    def test_annotated_without_json(self):
+        type_str = "typing.Optional[typing.Annotated[str, SomethingElse]]"
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(td.type, "str")
+        self.assertFalse(td.is_json)
+
+    def test_annotated_multiple_args(self):
+        type_str = "typing.Optional[typing.Annotated[list[int], Json, more_args]]"
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(td.type, "list[int]")
+        self.assertTrue(td.is_json)
+
+    def test_simple_optional(self):
+        type_str = "typing.Optional[uuid.UUID]"
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(td.type, "uuid.UUID")
+        self.assertFalse(td.is_json)
+
+    def test_whitespace_variations(self):
+        test_cases = [
+            ("typing.Optional[typing.Annotated[  str  ,  Json  ]]", "str", True),
+            ("typing.Optional[  typing.Annotated[dict,Json]]", "dict", True),
+            ("typing.Optional[  int  ]", "int", False),
+        ]
+
+        for type_str, expected_type, expected_json in test_cases:
+            with self.subTest(type_str=type_str):
+                td = TypeDescriptor(type_str, {})
+                self.assertEqual(td.type, expected_type)
+                self.assertEqual(td.is_json, expected_json)
+
+    def test_invalid_input(self):
+        test_cases = [
+            "typing.Optional[missing_bracket",
+            "typing.Optional[]",
+            "[without.optional]",
+        ]
+
+        for type_str in test_cases:
+            with self.subTest(type_str=type_str):
+                td = TypeDescriptor(type_str, {})
+                self.assertEqual(td.type, "")
+                self.assertFalse(td.is_json)
+
+    def test_repr(self):
+        type_str = "typing.Optional[typing.Annotated[str, Json]]"
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(repr(td), "TypeDescriptor(type='str', is_json=True)")
+
+    def test_instantiate_basic_types(self):
+        test_cases = [
+            ("str", "hello", "hello"),
+            ("int", "42", 42),
+            ("float", "3.14", 3.14),
+            ("bool", "true", True),
+        ]
+
+        for type_str, input_value, expected in test_cases:
+            with self.subTest(type_str=type_str, input_value=input_value):
+                result = _instantiate(type_str, input_value)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_list_types(self):
+        test_cases = [
+            ("typing.List[str]", "a,b,c", ["a", "b", "c"]),
+            ("typing.List[int]", "1,2,3", [1, 2, 3]),
+            ("typing.List[str]", json.dumps(["a", "b", "c"]), ["a", "b", "c"], True),
+            ("typing.List[int]", json.dumps([1, 2, 3]), [1, 2, 3], True),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_dict_types(self):
+        test_cases = [
+            ("typing.Dict", {"a": 1, "b": 2}, {"a": 1, "b": 2}),
+            ("typing.Dict", json.dumps({"a": 1, "b": 2}), {"a": 1, "b": 2}, True),
+            ("typing.Mapping", {"a": 1, "b": 2}, {"a": 1, "b": 2}),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_set_types(self):
+        test_cases = [
+            ("typing.Set[str]", "a,b,c", {"a", "b", "c"}),
+            ("typing.Set[int]", "1,2,3", {1, 2, 3}),
+            ("typing.Set[str]", json.dumps(["a", "b", "c"]), {"a", "b", "c"}, True),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_tuple_types(self):
+        test_cases = [
+            ("typing.Tuple[str, int]", ["hello", 42], ("hello", 42)),
+            ("typing.Tuple[str, int]", json.dumps(["hello", 42]), ("hello", 42), True),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_optional_types(self):
+        test_cases = [
+            ("typing.Optional[str]", "hello", "hello"),
+            ("typing.Optional[typing.List[str]]", "a,b,c", ["a", "b", "c"]),
+            (
+                "typing.Optional[typing.List[str]]",
+                json.dumps(["a", "b", "c"]),
+                ["a", "b", "c"],
+                True,
+            ),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_nested_types(self):
+        test_cases = [
+            (
+                "typing.List[typing.Dict]",
+                json.dumps([{"a": 1}, {"b": 2}]),
+                [{"a": 1}, {"b": 2}],
+                True,
+            ),
+            (
+                "typing.Optional[typing.List[typing.Dict]]",
+                json.dumps([{"a": 1}, {"b": 2}]),
+                [{"a": 1}, {"b": 2}],
+                True,
+            ),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_error_handling(self):
+        test_cases = [
+            ("typing.List[str]", "invalid json", True, json.JSONDecodeError),
+            ("typing.NonExistentType", "value", False, ImportError),
+        ]
+
+        for type_str, input_value, is_json, expected_exception in test_cases:
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                with self.assertRaises(expected_exception):
+                    _instantiate(type_str, input_value, is_json=is_json)
+
+    def test_instantiate_with_extra_init_data(self):
+
+        test_cases = [
+            (
+                "tests.adapter.test_type_descriptor.TestClass",
+                {"value": "test"},
+                {"extra": "extra_value"},
+                lambda x: x.value == "test" and x.extra == "extra_value",
+            ),
+        ]
+
+        for type_str, input_value, extra_init_data, validator in test_cases:
+            with self.subTest(type_str=type_str, input_value=input_value):
+                result = _instantiate(type_str, input_value, extra_init_data=extra_init_data)
+                self.assertTrue(validator(result))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/model/test_model.py

@@ -0,0 +1,220 @@
+import base64
+import json
+import unittest
+
+from amqp.model.model import (
+    AMQErrorMessage,
+    AMQRoute,
+    CleverMicroMessage,
+    DataMessage,
+    DataResponse,
+    ScalingRequest,
+    ScalingRequestAlert,
+)
+from amqp.model.snowflake_id import SnowflakeId
+
+
+class TestCleverMicroMessage(unittest.TestCase):
+    def setUp(self):
+        self.test_id = SnowflakeId()
+        self.test_headers = {"Content-Type": ["application/json"]}
+        self.test_trace_info = {"trace_id": "123"}
+        self.test_body = b"test data"
+        self.test_base64body = base64.b64encode(self.test_body)
+
+        self.message = CleverMicroMessage(
+            magic="A",
+            id=self.test_id,
+            m_field="GET",
+            d_field="test_domain",
+            p_field="test_path",
+            headers=self.test_headers,
+            trace_info=self.test_trace_info,
+            base64body=self.test_base64body,
+        )
+
+    def test_getters(self):
+        self.assertEqual(self.message.magic(), "A")
+        self.assertEqual(self.message.id(), self.test_id)
+        self.assertEqual(self.message.m_field(), "GET")
+        self.assertEqual(self.message.d_field(), "test_domain")
+        self.assertEqual(self.message.p_field(), "test_path")
+        self.assertEqual(self.message.headers(), self.test_headers)
+        self.assertEqual(self.message.trace_info(), self.test_trace_info)
+        self.assertEqual(self.message.base64body(), self.test_base64body)
+
+    def test_body_decoding(self):
+        self.assertEqual(self.message.body(), self.test_body)
+
+    def test_content_type(self):
+        self.assertEqual(self.message.content_type(), "application/json")
+
+        # Test default content type
+        message_no_content_type = CleverMicroMessage(
+            magic="A",
+            id=self.test_id,
+            m_field="GET",
+            d_field="test_domain",
+            p_field="test_path",
+            headers={},
+            trace_info={},
+            base64body=self.test_base64body,
+        )
+        self.assertEqual(
+            message_no_content_type.content_type(), CleverMicroMessage.DEFAULT_CONTENT_TYPE[0]
+        )
+
+
+class TestDataMessage(unittest.TestCase):
+    def setUp(self):
+        self.test_id = SnowflakeId()
+        self.test_headers = {"Content-Type": ["application/json"]}
+        self.test_trace_info = {"trace_id": "123"}
+        self.test_base64body = base64.b64encode(b"test data")
+
+        self.message = DataMessage(
+            magic="A",
+            id=self.test_id,
+            method="GET",
+            domain="testdomain",
+            path="testpath",
+            headers=self.test_headers,
+            trace_info=self.test_trace_info,
+            base64body=self.test_base64body,
+        )
+
+    def test_getters(self):
+        self.assertEqual(self.message.method(), "GET")
+        self.assertEqual(self.message.domain(), "testdomain")
+        self.assertEqual(self.message.path(), "testpath")
+
+    def test_routing_key(self):
+        self.assertEqual(self.message.routing_key(), "testdomain.testpath")
+
+
+class TestDataResponse(unittest.TestCase):
+    def setUp(self):
+        self.test_id = SnowflakeId()
+        self.test_trace_info = {"trace_id": "123"}
+        self.test_base64body = base64.b64encode(b"test data")
+
+        self.response = DataResponse(
+            magic="R",
+            id=self.test_id,
+            response_code="200",
+            content_type="application/json",
+            error="",
+            error_cause="",
+            headers={},
+            trace_info=self.test_trace_info,
+            base64body=self.test_base64body,
+        )
+
+    def test_getters(self):
+        self.assertEqual(self.response.response_code(), "200")
+        self.assertEqual(self.response.error(), "")
+        self.assertEqual(self.response.error_cause(), "")
+        self.assertEqual(self.response.content_type(), "application/json")
+
+
+class TestAMQRoute(unittest.TestCase):
+    def setUp(self):
+        self.route = AMQRoute(
+            component_name="test_component",
+            exchange="test_exchange",
+            queue="test_queue",
+            key="test_key",
+            timeout=1000,
+            valid_until=1735689600000,  # Some future timestamp
+        )
+
+    def test_str_representation(self):
+        expected = "test_component:test_exchange:test_queue:test_key:1000:1735689600000"
+        self.assertEqual(str(self.route), expected)
+
+    def test_equality(self):
+        same_route = AMQRoute(
+            component_name="test_component",
+            exchange="test_exchange",
+            queue="test_queue",
+            key="test_key",
+            timeout=1000,
+            valid_until=1735689600000,
+        )
+        different_route = AMQRoute(
+            component_name="other_component",
+            exchange="test_exchange",
+            queue="test_queue",
+            key="test_key",
+            timeout=1000,
+            valid_until=1735689600000,
+        )
+
+        self.assertEqual(self.route, same_route)
+        self.assertNotEqual(self.route, different_route)
+
+
+class TestScalingRequest(unittest.TestCase):
+    def setUp(self):
+        self.request = ScalingRequest(
+            service_id="test_service",
+            task_id="test_task",
+            max_availability=5,
+            current_availability=3,
+            request_type=ScalingRequestAlert.OVERLOAD,
+            version=1,
+        )
+
+    def test_to_json(self):
+        json_str = self.request.to_json()
+        data = json.loads(json_str)
+
+        self.assertEqual(data["serviceId"], "test_service")
+        self.assertEqual(data["taskId"], "test_task")
+        self.assertEqual(data["maxAvailability"], 5)
+        self.assertEqual(data["currentAvailability"], 3)
+        self.assertEqual(data["requestType"], "OVERLOAD")
+        self.assertEqual(data["version"], 1)
+
+    def test_from_json(self):
+        json_str = json.dumps(
+            {
+                "serviceId": "test_service",
+                "taskId": "test_task",
+                "maxAvailability": 5,
+                "currentAvailability": 3,
+                "requestType": "OVERLOAD",
+                "version": 1,
+            }
+        )
+
+        request = ScalingRequest.from_json(json_str)
+
+        self.assertEqual(request.service_id, "test_service")
+        self.assertEqual(request.task_id, "test_task")
+        self.assertEqual(request.max_availability, 5)
+        self.assertEqual(request.current_availability, 3)
+        self.assertEqual(request.request_type, ScalingRequestAlert.OVERLOAD)
+        self.assertEqual(request.version, 1)
+
+
+class TestAMQErrorMessage(unittest.TestCase):
+    def setUp(self):
+        self.error = AMQErrorMessage(
+            response_code=400,
+            error="Bad Request",
+            detail="Invalid input",
+            content_type="application/json",
+        )
+
+    def test_serialize(self):
+        serialized = self.error.serialize()
+        data = json.loads(serialized)
+
+        self.assertEqual(data["response_code"], 400)
+        self.assertEqual(data["error"], "Bad Request")
+        self.assertEqual(data["detail"], "Invalid input")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/model/test_snowflake_id.py

@@ -0,0 +1,48 @@
+from unittest import TestCase
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.model.snowflake_id import SnowflakeId
+
+
+class SnowflakeIdTest(TestCase):
+
+    def test_equals(self):
+        _id_1 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        _id_2 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        self.assertFalse(_id_1 == _id_2)
+        _id_3 = SnowflakeId(_id_1.get_bytes())
+        self.assertTrue(_id_1 == _id_3)
+
+    def test_lt(self):
+        _id_1 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        _id_2 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        self.assertTrue(_id_1 < _id_2)
+
+    def test_from_hex(self):
+        _factory = DataMessageFactory.get_instance(1)
+        _id_1 = _factory.generate_snowflake_id()
+        _id_1_ser = _id_1.as_string()
+        _id_2 = SnowflakeId.from_hex(_id_1_ser)
+        self.assertTrue(_id_1 == _id_2)
+        with self.assertRaises(RuntimeError) as context:
+            SnowflakeId.from_hex("100")
+        self.assertTrue(
+            f"MessageId is {SnowflakeId.SNOWFLAKE_ID_WIDTH * 8} bits wide, it is represented by HEX string "
+            in str(context.exception)
+        )
+
+    def test_get_bytes(self):
+        _factory = DataMessageFactory.get_instance(1)
+        _id_1 = _factory.generate_snowflake_id()
+        _id_1_ser = _id_1.get_bytes()
+        self.assertEqual(SnowflakeId.SNOWFLAKE_ID_WIDTH, len(_id_1_ser))
+
+    def test_get_bits(self):
+        _factory = DataMessageFactory.get_instance(1)
+        _id_1 = _factory.generate_snowflake_id()
+        self.assertEqual(2, len(_id_1.get_bits()))
+
+    def test_as_string(self):
+        _factory = DataMessageFactory.get_instance(1)
+        _id_1 = _factory.generate_snowflake_id()
+        self.assertEqual(2 * SnowflakeId.SNOWFLAKE_ID_WIDTH, len(_id_1.as_string()))

tests/rabbitmq/test_rabbit_mq_client.py

@@ -0,0 +1,209 @@
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from amqp.model.model import DataMessage
+from amqp.rabbitmq.rabbit_mq_client import RabbitMQClient
+
+
+class TestRabbitMQClient(IsolatedAsyncioTestCase):
+    def setUp(self):
+        self.mock_config = MagicMock()
+        self.mock_config.dispatch.amq_host = "localhost"
+        self.mock_config.dispatch.amq_port = 5672
+        self.mock_config.dispatch.rabbit_mq_user = "user"
+        self.mock_config.dispatch.rabbit_mq_password = "pass"
+        self.mock_config.dispatch.use_dlq = False
+        patcher = patch("amqp.rabbitmq.rabbit_mq_client.RouterConsumer")
+        self.addCleanup(patcher.stop)
+        self.mock_router = patcher.start()
+        self.client = RabbitMQClient(self.mock_config)
+
+    async def test_init_sets_up_channel_and_reply_exchange(self):
+        """Test that init sets up the channel, reply queue, and reply exchange."""
+        loop = MagicMock()
+        self.client.setup_amq_channel = AsyncMock()
+        self.client.router.init_routing_paths_consumer = AsyncMock()
+        self.client.router.get_reply_to_queue_name = MagicMock(return_value="reply_queue")
+        self.client.channel = MagicMock()
+        self.client.channel.declare_queue = AsyncMock()
+        self.client.channel.declare_exchange = AsyncMock()
+        queue_mock = MagicMock()
+        exchange_mock = MagicMock()
+        self.client.channel.declare_queue.return_value = queue_mock
+        self.client.channel.declare_exchange.return_value = exchange_mock
+        queue_mock.bind = AsyncMock()
+
+        result = await self.client.init(loop)
+
+        self.client.setup_amq_channel.assert_awaited_once_with(loop)
+        self.client.router.init_routing_paths_consumer.assert_awaited_once()
+        self.client.channel.declare_queue.assert_awaited_once()
+        self.client.channel.declare_exchange.assert_awaited_once()
+        queue_mock.bind.assert_awaited_once_with(exchange=exchange_mock, routing_key="reply_queue")
+        self.assertEqual(result, exchange_mock)
+
+    def test_cleanup_closes_channel_and_connection(self):
+        """Test that cleanup closes the channel and connection if open."""
+        self.client.router.cleanup = MagicMock()
+        self.client.router.is_open = MagicMock(return_value=True)
+        self.client.channel = MagicMock()
+        self.client.connection = MagicMock()
+        self.client.connection.is_closed = False
+        self.client.channel.close = MagicMock()
+        self.client.connection.close = MagicMock()
+
+        self.client.cleanup()
+
+        self.client.router.cleanup.assert_called_once()
+        self.client.channel.close.assert_called_once()
+        self.client.connection.close.assert_called_once()
+
+    def test_cleanup_handles_exceptions(self):
+        """Test that cleanup handles exceptions during close."""
+        self.client.router.cleanup = MagicMock()
+        self.client.router.is_open = MagicMock(return_value=True)
+        self.client.channel = MagicMock()
+        self.client.connection = MagicMock()
+        self.client.connection.is_closed = False
+        self.client.channel.close = MagicMock(side_effect=Exception("fail"))
+        self.client.connection.close = MagicMock()
+
+        # Should not raise
+        self.client.cleanup()
+
+    async def test_register_inbound_routes_registers_routes(self):
+        """Test that register_inbound_routes declares queues, exchanges, binds, and registers routes."""
+        self.client.router.is_open = MagicMock(return_value=True)
+        self.client.amq_configuration = self.mock_config
+        self.client.router.unwrap_route_list = MagicMock(
+            return_value=[
+                MagicMock(
+                    queue="q1",
+                    exchange="ex1",
+                    key="rk1",
+                    component_name="comp",
+                    timeout=0,
+                    valid_until=0,
+                )
+            ]
+        )
+        self.client.router.get_consume_queue_name = MagicMock(return_value="q1")
+        self.client.channel = MagicMock()
+        self.client.channel.declare_queue = AsyncMock()
+        self.client.channel.declare_exchange = AsyncMock()
+        queue_mock = MagicMock()
+        exchange_mock = MagicMock()
+        self.client.channel.declare_queue.return_value = queue_mock
+        self.client.channel.declare_exchange.return_value = exchange_mock
+        queue_mock.bind = AsyncMock()
+        queue_mock.consume = AsyncMock(return_value="ctag")
+        self.client.router.register_route = AsyncMock()
+
+        await self.client.register_inbound_routes("route_map", MagicMock())
+
+        self.client.channel.declare_queue.assert_awaited_once()
+        self.client.channel.declare_exchange.assert_awaited_once()
+        queue_mock.bind.assert_awaited_once_with(exchange_mock, "rk1")
+        queue_mock.consume.assert_awaited_once()
+        self.client.router.register_route.assert_awaited_once()
+
+    async def test_register_inbound_routes_channel_null(self):
+        """Test that register_inbound_routes logs a warning if channel is null."""
+        self.client.router.is_open = MagicMock(return_value=False)
+        with patch("amqp.rabbitmq.rabbit_mq_client.logging_warning") as mock_log:
+            await self.client.register_inbound_routes("route_map", MagicMock())
+            mock_log.assert_called()
+
+    async def test_register_data_reply_callback(self):
+        """Test that register_data_reply_callback consumes on reply_to_queue."""
+        queue_mock = MagicMock()
+        queue_mock.consume = AsyncMock(return_value="ctag")
+        queue_mock.name = "reply_queue"
+        self.client.reply_to_queue = queue_mock
+        with patch("amqp.rabbitmq.rabbit_mq_client.logging_info") as mock_log:
+            await self.client.register_data_reply_callback(MagicMock())
+            queue_mock.consume.assert_awaited_once()
+            mock_log.assert_called()
+
+    async def test_setup_amq_channel_sets_channel_and_qos(self):
+        """Test that setup_amq_channel sets up the connection, channel, and QoS."""
+        loop = MagicMock()
+        with patch("aio_pika.connect_robust", new_callable=AsyncMock) as mock_connect:
+            mock_connection = MagicMock()
+            mock_connect.return_value = mock_connection
+            mock_connection.channel = AsyncMock()
+            mock_channel = MagicMock()
+            mock_connection.channel.return_value = mock_channel
+            mock_channel.set_qos = AsyncMock()
+            self.client.router.init_internal_routing_paths = AsyncMock()
+            self.client.router.set_route_added_notifier = MagicMock()
+
+            await self.client.setup_amq_channel(loop)
+
+            mock_connect.assert_awaited_once()
+            mock_connection.channel.assert_awaited_once()
+            mock_channel.set_qos.assert_awaited_once_with(
+                prefetch_count=RabbitMQClient.PREFETCH_COUNT
+            )
+            self.client.router.init_internal_routing_paths.assert_awaited_once_with(mock_channel)
+            self.client.router.set_route_added_notifier.assert_called_once()
+
+    async def test_send_message_no_route(self):
+        """Test that send_message logs a warning if no route is found."""
+        self.client.router.find_route_by_message = MagicMock(return_value=None)
+        self.client.rpc_exchange = MagicMock()
+        message = MagicMock(spec=DataMessage)
+        with patch("amqp.rabbitmq.rabbit_mq_client.logging_warning") as mock_log:
+            await self.client.send_message(message)
+            mock_log.assert_called()
+
+    async def test_send_message_no_reply(self):
+        """Test that send_message publishes a message with no reply expected."""
+        route = MagicMock(timeout=0)
+        self.client.router.find_route_by_message = MagicMock(return_value=route)
+        self.client.rpc_exchange = AsyncMock()
+        self.client.router.get_routing_key = MagicMock(return_value="rk")
+        message = MagicMock(spec=DataMessage)
+        with patch(
+            "amqp.rabbitmq.rabbit_mq_client.DataMessageFactory.serialize", return_value=b"bytes"
+        ):
+            with patch("amqp.rabbitmq.rabbit_mq_client.logging_info") as mock_log:
+                await self.client.send_message(message)
+                self.client.rpc_exchange.publish.assert_awaited_once()
+                mock_log.assert_called()
+
+    async def test_send_message_rpc_call(self):
+        """Test that send_message publishes a message with reply expected (RPC)."""
+        route = MagicMock(timeout=1)
+        self.client.router.find_route_by_message = MagicMock(return_value=route)
+        self.client.rpc_exchange = AsyncMock()
+        self.client.router.get_routing_key = MagicMock(return_value="rk")
+        self.client.router.get_reply_to_queue_name = MagicMock(return_value="reply_q")
+        message = MagicMock(spec=DataMessage)
+        message.id.return_value = "id"
+        with patch(
+            "amqp.rabbitmq.rabbit_mq_client.DataMessageFactory.serialize", return_value=b"bytes"
+        ):
+            await self.client.send_message(message)
+            self.client.rpc_exchange.publish.assert_awaited_once()
+
+    async def test_send_message_exception(self):
+        """Test that send_message logs an error if an exception occurs."""
+        self.client.router.find_route_by_message = MagicMock(side_effect=Exception("fail"))
+        message = MagicMock(spec=DataMessage)
+        with patch("amqp.rabbitmq.rabbit_mq_client.logging_error") as mock_log:
+            await self.client.send_message(message)
+            mock_log.assert_called()
+
+    async def test_request_routes_delegates(self):
+        """Test that request_routes delegates to router."""
+        self.client.router.request_routes_from_adapters = AsyncMock()
+        await self.client.request_routes()
+        self.client.router.request_routes_from_adapters.assert_awaited_once()
+
+    def test_find_route_delegates(self):
+        """Test that find_route delegates to router."""
+        self.client.router.find_route = MagicMock(return_value="route")
+        result = self.client.find_route("domain", "path")
+        self.client.router.find_route.assert_called_once_with("domain", "path")
+        self.assertEqual(result, "route")

tests/requirements-test.txt

@@ -0,0 +1,4 @@
+pytest>=7.0.0
+pytest-cov>=4.0.0
+pytest-asyncio>=0.21.0
+aio-pika>=9.0.0