clevermicro/amq-adapter-python
14
0
Fork 0
Code Issues 16 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Added support for file download

Browse Source
This commit is contained in:
Stanislav Hejny
2025-04-09 19:23:26 +01:00
parent 6b420dee0a
commit 113dd38106
16 changed files with 250 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amqp/adapter/cleverthis_service_adapter.py
+92 -83
View File
@@ -2,25 +2,45 @@
The layer that interfaces with the CleverThis service that this Adapter is integrating with.
Methods here will invoke the actual CleverThis Service code, which are passed as Callable object.
"""
import asyncio
import inspect
import json
import logging
import re
import sys
import traceback
from re import Pattern
from typing import Dict, List, Tuple, Callable, Coroutine, AnyStr, Any
from asyncio import Future
from typing import Dict, List, Tuple, Callable, Coroutine, Any
from urllib.parse import parse_qs, urlparse
from xml.etree import ElementTree
from aio_pika.abc import AbstractRobustChannel
from aiohttp import ClientSession, ClientResponse
from dataclasses import dataclass
from fastapi import HTTPException
from starlette.responses import FileResponse
from amqp.adapter import pydantic_serializer
from amqp.adapter.data_response_factory import DataResponseFactory
from amqp.adapter.pydantic_serializer import JSONConverter
from amqp.config.amq_configuration import AMQConfiguration
from amqp.model.model import DataMessage, DataResponse
from amqp.service.multipart_parser import CleverMultiPartParser
from amqp.adapter.multipart_parser import CleverMultiPartParser, process_form_data
from amqp.adapter.file_uploader import publish_file_to_rabbitmq
@dataclass
class AMQMessage(DataMessage):
def __init__(self, data_message: DataMessage, channel: AbstractRobustChannel):
# Copy all fields from DataMessage
self.magic = data_message.magic
self.id = data_message.id
self.method = data_message.method
self.domain = data_message.domain
self.path = data_message.path
self.headers = data_message.headers
self.trace_info = data_message.trace_info
self.base64_body = data_message.base64_body
# Add the new field
self.channel = channel
def parse_request_body(message: DataMessage):
@@ -42,17 +62,22 @@ def parse_request_body(message: DataMessage):
try:
return json.loads(message.body())
except json.JSONDecodeError as e:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}")
_tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {e}")
raise ValueError(f"Invalid JSON: {e}")
# Parse URL-encoded form data
elif content_type == "application/x-www-form-urlencoded":
try:
return {k: v[0] if len(v) == 1 else v for k, v in parse_qs(message.body()).items()}
_form_data: dict = {k: v[0] if len(v) == 1 else v for k, v in parse_qs(message.body()).items()}
if len(_form_data) == 0 and len(message.body()) > 0:
_form_data = json.loads(message.body())
if _form_data['files'] and _form_data['form']:
_form_data = process_form_data(_form_data)
return _form_data
except Exception as e:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}")
_tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {e}")
raise ValueError(f"Invalid URL-encoded form data: {e}")
# Parse multipart/form-data
@@ -61,8 +86,8 @@ def parse_request_body(message: DataMessage):
parser: CleverMultiPartParser = CleverMultiPartParser(message)
return parser.form_data
except Exception as e:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}")
_tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {e}")
raise ValueError(f"Invalid multipart/form-data: {e}")
# Parse plain text
@@ -72,8 +97,8 @@ def parse_request_body(message: DataMessage):
# Parse XML
elif content_type == "application/xml":
try:
root = ElementTree.fromstring(message.body())
return {elem.tag: elem.text for elem in root}
_root = ElementTree.fromstring(message.body())
return {elem.tag: elem.text for elem in _root}
except ElementTree.ParseError as e:
raise ValueError(f"Invalid XML: {e}")
@@ -81,8 +106,22 @@ def parse_request_body(message: DataMessage):
raise ValueError(f"Unsupported Content-Type: {content_type}")
async def call_cleverthis_post_put_api(
message: DataMessage,
async def convert_file_response(obj: FileResponse, channel: AbstractRobustChannel) -> str:
"""
Converts a FileResponse object to a JSON string containing the filename and stream.
"""
_future: Future = asyncio.run_coroutine_threadsafe(
publish_file_to_rabbitmq(channel, obj.path), loop=channel.channel.loop)
return json.dumps({'files':[
{'filename': obj.filename,
'mediaType': 'application/octet-stream',
'streamName': _future.result()
}]
})
async def call_cleverthis_api(
message: AMQMessage,
args: Any,
api_method: Callable[[Any, Any], Coroutine],
success_code: int
@@ -98,19 +137,34 @@ async def call_cleverthis_post_put_api(
:return: DataResponse class
"""
try:
result: str = await api_method(**args)
return DataResponseFactory.of(
message.id, success_code, message.headers.get("Content-Type", "application/json"),
result.encode('utf-8'), message.trace_info
)
_verified_args = {}
_sig = inspect.signature(api_method)
_required_args = list(_sig.parameters.values())
#_required_args = getattr(api_method, '__annotations__', {})
for arg in _required_args:
if arg.name in args:
_verified_args[arg.name] = args[arg.name]
else:
logging.info(f"{api_method.__name__}: Missing required argument: {arg.name}")
logging.info(
f" [*] INVOKE ENDPOINT: {api_method.__name__}, ARGS: {', '.join(f'{k}={v}' for k, v in _verified_args.items())}")
_result: Any = await api_method(**_verified_args)
logging.info(f" [*] ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}")
_json_result, _content_type = await convert_file_response(_result, message.channel), 'application/octet-stream' \
if isinstance(_result, FileResponse) else _result, 'application/json'
_binary_result = _json_result.encode('utf-8') if hasattr(_json_result, 'encode') else \
_json_result.body if hasattr(_json_result, 'body') else \
_json_result.read() if hasattr(_json_result, 'read') else \
_json_result
return DataResponseFactory.of(message.id, success_code, _content_type, _binary_result, message.trace_info)
except HTTPException as http_error:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {http_error}")
_tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {http_error}")
return DataResponseFactory.of(message.id, http_error.status_code, "text/plain",
str(http_error.detail).encode('utf-8'), message.trace_info)
except Exception as error:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {error}")
_tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {error}")
return DataResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
@@ -126,61 +180,11 @@ def parse_get_url(url: str) -> Tuple[str, Dict[str, str]]:
- context_path: The path part of the URL (e.g., "/path")
- query_params_dict: Query parameters where values are str for single value or list of str for multivalue
"""
parsed = urlparse(url)
query_params = parse_qs(parsed.query)
_parsed = urlparse(url)
_query_params = parse_qs(_parsed.query)
# Convert values from lists to single values when there's only one value
simplified_params = {k: v[0] if len(v) == 1 else v for k, v in query_params.items()}
return parsed.path, simplified_params
async def call_cleverthis_get_api(
message: DataMessage,
pattern_or_data: Pattern| str | dict,
api_method: Callable[[Tuple[AnyStr | Any, ...], Any], Coroutine],
authenticated_user: Any
) -> DataResponse:
"""
Handles GET requests, which do not contain a body, but may have path variables and extra values
in the query string.
invokes provided Callable that is expected to be a GET API method, and returns the response as DataResponse
:param message: DataMessage containing the details of the call
:param pattern_or_data: If there are path variables, use this pattern to extract them
:param api_method: Callable - the method to invoke.
:param authenticated_user: Authenticated user. The actual class depends on the service
:return:
"""
try:
args = None
if isinstance(pattern_or_data, dict):
args = pattern_or_data
elif isinstance(pattern_or_data, Pattern):
path, args = pydantic_serializer.parse_url_query(message.path)
match = pattern_or_data.match(path)
if match:
args.update(match.groupdict())
else:
_msg = f"Unexpected URL: {message.path}, expected pattern: {pattern_or_data}"
return DataResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info)
else:
path, args = pydantic_serializer.parse_url_query(message.path)
match = re.search(pattern_or_data, path)
if match:
args.update(match.groupdict())
else:
_msg = f"Unexpected URL: {message.path}, expected pattern: {pattern_or_data}"
return DataResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info)
crt: Coroutine = api_method(args, authenticated_user)
result: str = await crt
return DataResponseFactory.of(message.id, 200, "text/plain", result.encode('utf-8'), message.trace_info)
except HTTPException as http_error:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {http_error}")
return DataResponseFactory.of(message.id, http_error.status_code, "text/plain",
str(http_error.detail).encode('utf-8'), message.trace_info)
except Exception as error:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {error}")
return DataResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
_simplified_params = {k: v[0] if len(v) == 1 else v for k, v in _query_params.items()}
return _parsed.path, _simplified_params
# ============= CleverThisServiceAdapter =============
@@ -192,6 +196,11 @@ class CleverThisServiceAdapter:
and override the methods to handle the specific API calls.
"""
def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession = None):
"""
Initializes the CleverThisServiceAdapter with the given AMQ configuration and optional HTTP session.
The HTTP session is for the case when the adapter uses loose coupling with the CleverThis service via
HTTP REST API.
"""
self.amq_configuration = amq_configuration
self.session = session
self.service_host = self.amq_configuration.dispatch.service_host
@@ -218,14 +227,14 @@ class CleverThisServiceAdapter:
"""
return None
async def on_message(self, message) -> DataResponse:
async def on_message(self, message: AMQMessage) -> DataResponse:
"""
Called when a message is received from the AMQP.
:param message:
:return: AMQP response class with operation status code and optional error details.
"""
_auth_user: Any | None = None
amq_response: DataResponse | None = None
amq_response: Any | None = None
if self.require_authenticated_user:
_auth: str = message.headers.get('Authorization', '')
@@ -317,7 +326,7 @@ class CleverThisServiceAdapter:
)
async def handle_possible_form(self, message: DataMessage, request_coroutine, auth_user: Any) -> DataResponse:
args = parse_request_body(message)
_args = parse_request_body(message)
return await request_coroutine
async def head(self, message: DataMessage, auth_user: Any) -> DataResponse: