clevermicro/amq-adapter-python
14
0
Fork 0
Code Issues 16 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat/#1 - applied 'black' formatting

Browse Source
This commit is contained in:
Stanislav Hejny
2025-04-18 23:49:10 +01:00
parent 04977e5ccc
commit 1de7afe4f5
35 changed files with 1239 additions and 563 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amqp/adapter/cleverthis_service_adapter.py
+155 -69
View File
@@ -2,6 +2,7 @@
The layer that interfaces with the CleverThis service that this Adapter is integrating with.
Methods here will invoke the actual CleverThis Service code, which are passed as Callable object.
"""
import asyncio
import inspect
import json
@@ -13,7 +14,12 @@ from typing import Dict, List, Tuple, Callable, Coroutine, Any
from urllib.parse import parse_qs, urlparse
from xml.etree import ElementTree
from aio_pika.abc import AbstractRobustChannel, AbstractRobustConnection, AbstractRobustExchange, AbstractExchange
from aio_pika.abc import (
AbstractRobustChannel,
AbstractRobustConnection,
AbstractRobustExchange,
AbstractExchange,
)
from aiohttp import ClientSession, ClientResponse
from dataclasses import dataclass
from fastapi import HTTPException
@@ -28,6 +34,7 @@ from amqp.adapter.file_uploader import publish_file_to_rabbitmq
from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
from IPython.core import debugger
debug = debugger.Pdb().set_trace
@@ -50,7 +57,9 @@ class AMQMessage(DataMessage):
self.base64_body = data_message.base64_body
# Add the new field
self.connection = connection
self.extra_data = data_message.extra_data if hasattr(data_message, 'extra_data') else {}
self.extra_data = (
data_message.extra_data if hasattr(data_message, "extra_data") else {}
)
def parse_get_url(url: str) -> Tuple[str, Dict[str, str]]:
@@ -68,7 +77,9 @@ def parse_get_url(url: str) -> Tuple[str, Dict[str, str]]:
_parsed = urlparse(url)
_query_params = parse_qs(_parsed.query)
# Convert values from lists to single values when there's only one value
_simplified_params = {k: v[0] if len(v) == 1 else v for k, v in _query_params.items()}
_simplified_params = {
k: v[0] if len(v) == 1 else v for k, v in _query_params.items()
}
return _parsed.path, _simplified_params
@@ -82,7 +93,7 @@ def parse_request_body(message: DataMessage):
Returns:
dict: Parsed data as a dictionary.
"""
_content_type = message.headers.get('Content-Type', '')
_content_type = message.headers.get("Content-Type", "")
if not _content_type:
raise ValueError("Content-Type header is required")
content_type = _content_type[0]
@@ -97,10 +108,13 @@ def parse_request_body(message: DataMessage):
# Parse URL-encoded form data
elif content_type == "application/x-www-form-urlencoded":
try:
_form_data: dict = {k: v[0] if len(v) == 1 else v for k, v in parse_qs(message.body()).items()}
_form_data: dict = {
k: v[0] if len(v) == 1 else v
for k, v in parse_qs(message.body()).items()
}
if len(_form_data) == 0 and len(message.body()) > 0:
_form_data = json.loads(message.body())
if _form_data['files'] and _form_data['form']:
if _form_data["files"] and _form_data["form"]:
_form_data = process_form_data(_form_data)
return _form_data
except Exception as e:
@@ -135,14 +149,15 @@ def parse_request_body(message: DataMessage):
async def _convert_file_response(
obj: FileResponse,
connection: AbstractRobustConnection,
loop: AbstractEventLoop | None
loop: AbstractEventLoop | None,
) -> str:
"""
Converts a FileResponse object to a JSON string containing the filename and stream name.
The file content itself is transmitted through the named stream (which is implemented as temporary dedicated queue.
"""
async def _wrap_amq_api_calls(
connection: AbstractRobustConnection
connection: AbstractRobustConnection,
) -> tuple[AbstractRobustChannel, AbstractExchange, str]:
# Wrap calls to RabbitMQ API to make it Awaitable, so it can be run thread-safe with correct event loop
# This is needed because the RabbitMQ API is not thread-safe and needs to be called in the context of
@@ -150,39 +165,53 @@ async def _convert_file_response(
_channel = await connection.channel()
_exchange_name: str = AMQ_REPLY_TO_EXCHANGE
_exchange = await _channel.get_exchange(name=_exchange_name, ensure=True)
_queue = await _channel.declare_queue(exclusive=False) # Declare a unique, non-exclusive queue
_queue = await _channel.declare_queue(
exclusive=False
) # Declare a unique, non-exclusive queue
_queue_name = _queue.name
await _queue.bind(
exchange=_exchange_name,
routing_key=_queue_name, # Use queue name as routing key
)
logging.debug(f" [*] Publishing to exchange: {_exchange_name}, q: {_queue_name}")
logging.debug(
f" [*] Publishing to exchange: {_exchange_name}, q: {_queue_name}"
)
return _channel, _exchange, _queue_name
# All calls to RabbitMQ API have to be called threadsafe, because those appear to be attached to parent
# event loop, and need to execute at that event loop, not the current one
_future = asyncio.run_coroutine_threadsafe(_wrap_amq_api_calls(connection), loop)
while not _future.done():
await asyncio.sleep(0.1) # allow coroutine to execute. calling _future.result() will block the entire thread and the coroutine will never execute
await asyncio.sleep(
0.1
) # allow coroutine to execute. calling _future.result() will block the entire thread and the coroutine will never execute
_channel, _exchange, _routing_key = _future.result()
# publish_file also needs RabbitMQ API, ensure the loop is passed as argument
(_stream_name, _file_size) = await publish_file_to_rabbitmq(_exchange, obj.path, _routing_key, loop)
(_stream_name, _file_size) = await publish_file_to_rabbitmq(
_exchange, obj.path, _routing_key, loop
)
# as above, but luckily this time we don't need to wait for the result
_future = asyncio.run_coroutine_threadsafe(_channel.close(), loop)
return json.dumps({'files': [
{'filename': obj.filename,
'mediaType': 'application/octet-stream',
'streamName': _stream_name,
'fileSize': _file_size
}]
})
return json.dumps(
{
"files": [
{
"filename": obj.filename,
"mediaType": "application/octet-stream",
"streamName": _stream_name,
"fileSize": _file_size,
}
]
}
)
def is_likely_json(text: str) -> bool:
""" Quick test to see if should convert the response string to JSON format. """
"""Quick test to see if should convert the response string to JSON format."""
_text = text.strip()
return (_text.startswith('{') and _text.endswith('}')) or \
(_text.startswith('[') and _text.endswith(']'))
return (_text.startswith("{") and _text.endswith("}")) or (
_text.startswith("[") and _text.endswith("]")
)
async def call_cleverthis_api(
@@ -190,7 +219,7 @@ async def call_cleverthis_api(
args: Any,
api_method: Callable[[Any, Any], Coroutine],
success_code: int,
loop: AbstractEventLoop | None = None
loop: AbstractEventLoop | None = None,
) -> DataResponse:
"""
Handles POST and PUT requests, which typically contain possibly complex payload data, passed as a dictionary.
@@ -220,41 +249,69 @@ async def call_cleverthis_api(
else:
_verified_args[arg.name] = args[arg.name]
else:
logging.info(f"{api_method.__name__}: Missing required argument: {arg.name}")
logging.info(
f"{api_method.__name__}: Missing required argument: {arg.name}"
)
logging.info(
f" [*] INVOKE ENDPOINT: {api_method.__name__}, ARGS: {', '.join(f'{k}={v}' for k, v in _verified_args.items())}")
f" [*] INVOKE ENDPOINT: {api_method.__name__}, ARGS: {', '.join(f'{k}={v}' for k, v in _verified_args.items())}"
)
_result: Any = await api_method(**_verified_args)
logging.info(f" [*] ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}")
logging.info(
f" [*] ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}"
)
if isinstance(_result, FileResponse):
_json_result, _content_type = await _convert_file_response(
_result, message.connection, loop
), 'application/octet-stream'
_json_result, _content_type = (
await _convert_file_response(_result, message.connection, loop),
"application/octet-stream",
)
else:
_json_result = _result
_content_type = 'application/json'
_content_type = "application/json"
_binary_result = _json_result.encode('utf-8') if hasattr(
_json_result, 'encode'
) else _json_result.body if hasattr(
_json_result, 'body'
) else _json_result.read() if hasattr(_json_result, 'read') else _json_result
logging.info(f" [x] ALL DONE in call_cleverthis_api id:{message.id}, resp code:{success_code}")
return DataResponseFactory.of(message.id, success_code, _content_type, _binary_result, message.trace_info)
_binary_result = (
_json_result.encode("utf-8")
if hasattr(_json_result, "encode")
else (
_json_result.body
if hasattr(_json_result, "body")
else (
_json_result.read()
if hasattr(_json_result, "read")
else _json_result
)
)
)
logging.info(
f" [x] ALL DONE in call_cleverthis_api id:{message.id}, resp code:{success_code}"
)
return DataResponseFactory.of(
message.id, success_code, _content_type, _binary_result, message.trace_info
)
except HTTPException as http_error:
_content = str(http_error.detail)
if not is_likely_json(_content):
_content = '{"error": "' + _content + '"}'
logging_error(f"Service REST endpoint invocation failed: {http_error}")
return DataResponseFactory.of(message.id, http_error.status_code, "application/json",
_content.encode('utf-8'), message.trace_info)
return DataResponseFactory.of(
message.id,
http_error.status_code,
"application/json",
_content.encode("utf-8"),
message.trace_info,
)
except Exception as error:
logging_error(f"Service endpoint invocation failed: {error}")
_content = str(error)
if not is_likely_json(_content):
_content = '{"error": "' + _content + '"}'
return DataResponseFactory.of(message.id, 500, "application/json",
_content.encode('utf-8'), message.trace_info)
return DataResponseFactory.of(
message.id,
500,
"application/json",
_content.encode("utf-8"),
message.trace_info,
)
# ============= CleverThisServiceAdapter =============
@@ -266,7 +323,9 @@ class CleverThisServiceAdapter:
and override the methods to handle the specific API calls.
"""
def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession = None):
def __init__(
self, amq_configuration: AMQConfiguration, session: ClientSession = None
):
"""
Initializes the CleverThisServiceAdapter with the given AMQ configuration and optional HTTP session.
The HTTP session is for the case when the adapter uses loose coupling with the CleverThis service via
@@ -277,8 +336,13 @@ class CleverThisServiceAdapter:
self.service_host = self.amq_configuration.dispatch.service_host
self.service_port = self.amq_configuration.dispatch.service_port
self.loop: AbstractEventLoop | None = None
self.require_authenticated_user = eval(
self.amq_configuration.amq_adapter.require_authenticated_user.capitalize()) if self.amq_configuration.amq_adapter.require_authenticated_user.strip() else True
self.require_authenticated_user = (
eval(
self.amq_configuration.amq_adapter.require_authenticated_user.capitalize()
)
if self.amq_configuration.amq_adapter.require_authenticated_user.strip()
else True
)
def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
"""
@@ -287,9 +351,9 @@ class CleverThisServiceAdapter:
return: The content type of the message.
"""
for header, values in message_headers.items():
if header.lower() == 'content-type':
return values[0].split(';')[0]
return 'application/json'
if header.lower() == "content-type":
return values[0].split(";")[0]
return "application/json"
async def get_current_user(self, token: str) -> object:
"""
@@ -310,37 +374,46 @@ class CleverThisServiceAdapter:
_amq_response: Any | None = None
logging.debug(
f' [*] on_message: _auth_user={_auth_user}, require_authenticated_user={self.require_authenticated_user}')
f" [*] on_message: _auth_user={_auth_user}, require_authenticated_user={self.require_authenticated_user}"
)
if self.require_authenticated_user:
_auth: str = message.headers.get('Authorization', '')
_auth: str = message.headers.get("Authorization", "")
logging.info(f"Auth: {_auth}")
if isinstance(_auth, List):
_auth = _auth[0]
if 'Bearer' in _auth:
if "Bearer" in _auth:
try:
token = _auth.split(' ')[1]
logging.warn(f'Token: {token}')
token = _auth.split(" ")[1]
logging.warn(f"Token: {token}")
_auth_user = await self.get_current_user(token)
except Exception as error:
logging_error(f"on_message: Error getting user: {error}")
if _auth_user or not self.require_authenticated_user or message.path.endswith('/login'):
if (
_auth_user
or not self.require_authenticated_user
or message.path.endswith("/login")
):
method = message.method.lower()
if method == 'get':
if method == "get":
_amq_response = await self.get(message, _auth_user)
elif method == 'put':
elif method == "put":
_amq_response = await self.put(message, _auth_user)
elif method == 'post':
elif method == "post":
_amq_response = await self.post(message, _auth_user)
elif method == 'head':
elif method == "head":
_amq_response = await self.head(message, _auth_user)
elif method == 'delete':
elif method == "delete":
_amq_response = await self.delete(message, _auth_user)
else:
raise ValueError(f"Unexpected HTTP method: {message.method}")
else:
_amq_response = DataResponseFactory.of(message.id, 401, "text/plain", b"Unauthorized", message.trace_info)
logging.info(f" [x] ALL DONE in on_message id:{message.id}, resp code:{_amq_response.response_code}")
_amq_response = DataResponseFactory.of(
message.id, 401, "text/plain", b"Unauthorized", message.trace_info
)
logging.info(
f" [x] ALL DONE in on_message id:{message.id}, resp code:{_amq_response.response_code}"
)
return _amq_response
# ==================================================================================================
@@ -359,14 +432,25 @@ class CleverThisServiceAdapter:
# unmatched path - try to invoke the service directly on this path
url = f"http://{self.service_host}:{self.service_port}{message.path}"
headers = {**message.headers, **message.trace_info}
logging.info(f" [*] SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}")
logging.info(
f" [*] SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}"
)
if self.session:
_http_resp: ClientResponse = await self.session.get(url, headers=headers)
return DataResponseFactory.of(message.id, _http_resp.status, _http_resp.content_type,
await _http_resp.read(),
message.trace_info)
return DataResponseFactory.of(message.id, 404, "text/plain",
str("Destination location does not exists").encode('utf-8'), message.trace_info)
return DataResponseFactory.of(
message.id,
_http_resp.status,
_http_resp.content_type,
await _http_resp.read(),
message.trace_info,
)
return DataResponseFactory.of(
message.id,
404,
"text/plain",
str("Destination location does not exists").encode("utf-8"),
message.trace_info,
)
async def put(self, message: DataMessage, auth_user: Any) -> DataResponse:
"""
@@ -383,7 +467,7 @@ class CleverThisServiceAdapter:
data=message.body,
headers={**message.headers, **message.trace_info},
),
auth_user
auth_user,
)
async def post(self, message: DataMessage, auth_user: Any) -> DataResponse:
@@ -401,10 +485,12 @@ class CleverThisServiceAdapter:
data=message.body,
headers={**message.headers, **message.trace_info},
),
auth_user
auth_user,
)
async def handle_possible_form(self, message: DataMessage, request_coroutine, auth_user: Any) -> DataResponse:
async def handle_possible_form(
self, message: DataMessage, request_coroutine, auth_user: Any
) -> DataResponse:
_args = parse_request_body(message)
return await request_coroutine