feat/#1 - refactor code for better design and testability
This commit was merged in pull request #11.
This commit is contained in:
Stanislav Hejny
@@ -7,17 +7,12 @@ import asyncio
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import traceback
|
||||
from asyncio import AbstractEventLoop
|
||||
from typing import Dict, List, Tuple, Callable, Coroutine, Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from xml.etree import ElementTree
|
||||
from typing import Dict, List, Callable, Coroutine, Any
|
||||
|
||||
from aio_pika.abc import (
|
||||
AbstractRobustChannel,
|
||||
AbstractRobustConnection,
|
||||
AbstractRobustExchange,
|
||||
AbstractExchange,
|
||||
)
|
||||
from aiohttp import ClientSession, ClientResponse
|
||||
@@ -25,18 +20,14 @@ from dataclasses import dataclass
|
||||
from fastapi import HTTPException
|
||||
from starlette.responses import FileResponse
|
||||
|
||||
from amqp.adapter.data_parser import AMQDataParser
|
||||
from amqp.adapter.data_response_factory import DataResponseFactory
|
||||
from amqp.adapter.file_handler import FileHandler, StreamingFileHandler
|
||||
from amqp.adapter.logging_utils import logging_error
|
||||
from amqp.config.amq_configuration import AMQConfiguration
|
||||
from amqp.model.model import DataMessage, DataResponse
|
||||
from amqp.adapter.multipart_parser import CleverMultiPartParser, process_form_data
|
||||
from amqp.adapter.file_uploader import publish_file_to_rabbitmq
|
||||
from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
|
||||
|
||||
from IPython.core import debugger
|
||||
|
||||
debug = debugger.Pdb().set_trace
|
||||
|
||||
|
||||
@dataclass
|
||||
class AMQMessage(DataMessage):
|
||||
@@ -62,93 +53,10 @@ class AMQMessage(DataMessage):
|
||||
)
|
||||
|
||||
|
||||
def parse_get_url(url: str) -> Tuple[str, Dict[str, str]]:
|
||||
"""
|
||||
Parses an HTTP GET URL and returns (context_path, query_params_dict)
|
||||
|
||||
Args:
|
||||
url: The URL string to parse (e.g., "http://example.com/path?foo=1&bar=2&foo=3")
|
||||
|
||||
Returns:
|
||||
tuple: (context_path, query_params_dict)
|
||||
- context_path: The path part of the URL (e.g., "/path")
|
||||
- query_params_dict: Query parameters where values are str for single value or list of str for multivalue
|
||||
"""
|
||||
_parsed = urlparse(url)
|
||||
_query_params = parse_qs(_parsed.query)
|
||||
# Convert values from lists to single values when there's only one value
|
||||
_simplified_params = {
|
||||
k: v[0] if len(v) == 1 else v for k, v in _query_params.items()
|
||||
}
|
||||
return _parsed.path, _simplified_params
|
||||
|
||||
|
||||
def parse_request_body(message: DataMessage):
|
||||
"""
|
||||
Parses the HTTP POST request body based on the MIME type.
|
||||
This is typically called fom the concrete CleverThis Service to extract parameters from the message body.
|
||||
Args:
|
||||
message (DataMessage): Contains raw request and MIME headers.
|
||||
|
||||
Returns:
|
||||
dict: Parsed data as a dictionary.
|
||||
"""
|
||||
_content_type = message.headers.get("Content-Type", "")
|
||||
if not _content_type:
|
||||
raise ValueError("Content-Type header is required")
|
||||
content_type = _content_type[0]
|
||||
# Parse JSON
|
||||
if content_type == "application/json":
|
||||
try:
|
||||
return json.loads(message.body())
|
||||
except json.JSONDecodeError as e:
|
||||
logging_error(f"Invalid JSON: {e}")
|
||||
raise ValueError(f"Invalid JSON: {e}")
|
||||
|
||||
# Parse URL-encoded form data
|
||||
elif content_type == "application/x-www-form-urlencoded":
|
||||
try:
|
||||
_form_data: dict = {
|
||||
k: v[0] if len(v) == 1 else v
|
||||
for k, v in parse_qs(message.body()).items()
|
||||
}
|
||||
if len(_form_data) == 0 and len(message.body()) > 0:
|
||||
_form_data = json.loads(message.body())
|
||||
if _form_data["files"] and _form_data["form"]:
|
||||
_form_data = process_form_data(_form_data)
|
||||
return _form_data
|
||||
except Exception as e:
|
||||
logging_error(f"Invalid URL-encoded form data: {e}")
|
||||
raise ValueError(f"Invalid URL-encoded form data: {e}")
|
||||
|
||||
# Parse multipart/form-data
|
||||
elif content_type.startswith("multipart/form-data"):
|
||||
try:
|
||||
parser: CleverMultiPartParser = CleverMultiPartParser(message)
|
||||
return parser.form_data
|
||||
except Exception as e:
|
||||
logging_error(f"Invalid multipart/form-data: {e}")
|
||||
raise ValueError(f"Invalid multipart/form-data: {e}")
|
||||
|
||||
# Parse plain text
|
||||
elif content_type == "text/plain":
|
||||
return {"text": message.body()}
|
||||
|
||||
# Parse XML
|
||||
elif content_type == "application/xml":
|
||||
try:
|
||||
_root = ElementTree.fromstring(message.body())
|
||||
return {elem.tag: elem.text for elem in _root}
|
||||
except ElementTree.ParseError as e:
|
||||
raise ValueError(f"Invalid XML: {e}")
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported Content-Type: {content_type}")
|
||||
|
||||
|
||||
async def _convert_file_response(
|
||||
obj: FileResponse,
|
||||
connection: AbstractRobustConnection,
|
||||
file_handler: FileHandler,
|
||||
loop: AbstractEventLoop | None,
|
||||
) -> str:
|
||||
"""
|
||||
@@ -187,7 +95,7 @@ async def _convert_file_response(
|
||||
) # allow coroutine to execute. calling _future.result() will block the entire thread and the coroutine will never execute
|
||||
_channel, _exchange, _routing_key = _future.result()
|
||||
# publish_file also needs RabbitMQ API, ensure the loop is passed as argument
|
||||
(_stream_name, _file_size) = await publish_file_to_rabbitmq(
|
||||
(_stream_name, _file_size) = await file_handler.publish_file(
|
||||
_exchange, obj.path, _routing_key, loop
|
||||
)
|
||||
# as above, but luckily this time we don't need to wait for the result
|
||||
@@ -214,106 +122,6 @@ def is_likely_json(text: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
async def call_cleverthis_api(
|
||||
message: AMQMessage,
|
||||
args: Any,
|
||||
api_method: Callable[[Any, Any], Coroutine],
|
||||
success_code: int,
|
||||
loop: AbstractEventLoop | None = None,
|
||||
) -> DataResponse:
|
||||
"""
|
||||
Handles POST and PUT requests, which typically contain possibly complex payload data, passed as a dictionary.
|
||||
Invokes the provided API method with the given arguments and authenticated user,
|
||||
and returns the response as DataResponse.
|
||||
:param message: DataMessage containing the details of the call and the payload
|
||||
:param args: map of the arguments to pass to the API method
|
||||
:param api_method: Callable - the method to invoke. Last parameter is the authenticated user
|
||||
:param success_code: HTTP code to be returned when operation suceeds
|
||||
:return: DataResponse class
|
||||
"""
|
||||
try:
|
||||
_verified_args = {}
|
||||
_sig = inspect.signature(api_method)
|
||||
_required_args = list(_sig.parameters.values())
|
||||
# _required_args = getattr(api_method, '__annotations__', {})
|
||||
for arg in _required_args:
|
||||
if arg.name in args:
|
||||
# convert to another type if arg is not a str
|
||||
if arg.annotation == int:
|
||||
_verified_args[arg.name] = int(args[arg.name])
|
||||
elif arg.annotation == float:
|
||||
_verified_args[arg.name] = float(args[arg.name])
|
||||
elif arg.annotation == bool:
|
||||
_verified_args[arg.name] = bool(args[arg.name])
|
||||
# TODO: more?
|
||||
else:
|
||||
_verified_args[arg.name] = args[arg.name]
|
||||
else:
|
||||
logging.info(
|
||||
f"{api_method.__name__}: Missing required argument: {arg.name}"
|
||||
)
|
||||
logging.info(
|
||||
f" [*] INVOKE ENDPOINT: {api_method.__name__}, ARGS: {', '.join(f'{k}={v}' for k, v in _verified_args.items())}"
|
||||
)
|
||||
_result: Any = await api_method(**_verified_args)
|
||||
logging.info(
|
||||
f" [*] ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}"
|
||||
)
|
||||
|
||||
if isinstance(_result, FileResponse):
|
||||
_json_result, _content_type = (
|
||||
await _convert_file_response(_result, message.connection, loop),
|
||||
"application/octet-stream",
|
||||
)
|
||||
else:
|
||||
_json_result = _result
|
||||
_content_type = "application/json"
|
||||
|
||||
_binary_result = (
|
||||
_json_result.encode("utf-8")
|
||||
if hasattr(_json_result, "encode")
|
||||
else (
|
||||
_json_result.body
|
||||
if hasattr(_json_result, "body")
|
||||
else (
|
||||
_json_result.read()
|
||||
if hasattr(_json_result, "read")
|
||||
else _json_result
|
||||
)
|
||||
)
|
||||
)
|
||||
logging.info(
|
||||
f" [x] ALL DONE in call_cleverthis_api id:{message.id}, resp code:{success_code}"
|
||||
)
|
||||
return DataResponseFactory.of(
|
||||
message.id, success_code, _content_type, _binary_result, message.trace_info
|
||||
)
|
||||
except HTTPException as http_error:
|
||||
_content = str(http_error.detail)
|
||||
if not is_likely_json(_content):
|
||||
_content = '{"error": "' + _content + '"}'
|
||||
logging_error(f"Service REST endpoint invocation failed: {http_error}")
|
||||
return DataResponseFactory.of(
|
||||
message.id,
|
||||
http_error.status_code,
|
||||
"application/json",
|
||||
_content.encode("utf-8"),
|
||||
message.trace_info,
|
||||
)
|
||||
except Exception as error:
|
||||
logging_error(f"Service endpoint invocation failed: {error}")
|
||||
_content = str(error)
|
||||
if not is_likely_json(_content):
|
||||
_content = '{"error": "' + _content + '"}'
|
||||
return DataResponseFactory.of(
|
||||
message.id,
|
||||
500,
|
||||
"application/json",
|
||||
_content.encode("utf-8"),
|
||||
message.trace_info,
|
||||
)
|
||||
|
||||
|
||||
# ============= CleverThisServiceAdapter =============
|
||||
class CleverThisServiceAdapter:
|
||||
"""
|
||||
@@ -336,13 +144,23 @@ class CleverThisServiceAdapter:
|
||||
self.service_host = self.amq_configuration.dispatch.service_host
|
||||
self.service_port = self.amq_configuration.dispatch.service_port
|
||||
self.loop: AbstractEventLoop | None = None
|
||||
self.require_authenticated_user = (
|
||||
eval(
|
||||
self.amq_configuration.amq_adapter.require_authenticated_user.capitalize()
|
||||
try:
|
||||
self.require_authenticated_user = (
|
||||
self.amq_configuration.amq_adapter.require_authenticated_user
|
||||
and isinstance(
|
||||
self.amq_configuration.amq_adapter.require_authenticated_user, bool
|
||||
)
|
||||
or eval(
|
||||
str(self.amq_configuration.amq_adapter.require_authenticated_user)
|
||||
.strip()
|
||||
.capitalize()
|
||||
or "True"
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
self.require_authenticated_user = (
|
||||
True # Enforce authenticated user as default
|
||||
)
|
||||
if self.amq_configuration.amq_adapter.require_authenticated_user.strip()
|
||||
else True
|
||||
)
|
||||
|
||||
def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
|
||||
"""
|
||||
@@ -384,7 +202,7 @@ class CleverThisServiceAdapter:
|
||||
if "Bearer" in _auth:
|
||||
try:
|
||||
token = _auth.split(" ")[1]
|
||||
logging.warn(f"Token: {token}")
|
||||
logging.warning(f"Token: {token}")
|
||||
_auth_user = await self.get_current_user(token)
|
||||
except Exception as error:
|
||||
logging_error(f"on_message: Error getting user: {error}")
|
||||
@@ -491,7 +309,7 @@ class CleverThisServiceAdapter:
|
||||
async def handle_possible_form(
|
||||
self, message: DataMessage, request_coroutine, auth_user: Any
|
||||
) -> DataResponse:
|
||||
_args = parse_request_body(message)
|
||||
_args = AMQDataParser.parse_request_body(message)
|
||||
return await request_coroutine
|
||||
|
||||
async def head(self, message: DataMessage, auth_user: Any) -> DataResponse:
|
||||
@@ -513,3 +331,112 @@ class CleverThisServiceAdapter:
|
||||
:return: DataResponse to be sent back to the API Gateway via AMQP
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
async def call_cleverthis_api(
|
||||
message: AMQMessage,
|
||||
args: Any,
|
||||
api_method: Callable[[Any, Any], Coroutine],
|
||||
success_code: int,
|
||||
loop: AbstractEventLoop | None = None,
|
||||
) -> DataResponse:
|
||||
"""
|
||||
Handles POST and PUT requests, which typically contain possibly complex payload data, passed as a dictionary.
|
||||
Invokes the provided API method with the given arguments and authenticated user,
|
||||
and returns the response as DataResponse.
|
||||
:param message: DataMessage containing the details of the call and the payload
|
||||
:param args: map of the arguments to pass to the API method
|
||||
:param api_method: Callable - the method to invoke. Last parameter is the authenticated user
|
||||
:param success_code: HTTP code to be returned when operation suceeds
|
||||
:return: DataResponse class
|
||||
"""
|
||||
try:
|
||||
_verified_args = {}
|
||||
_sig = inspect.signature(api_method)
|
||||
_required_args = list(_sig.parameters.values())
|
||||
# _required_args = getattr(api_method, '__annotations__', {})
|
||||
for arg in _required_args:
|
||||
if arg.name in args:
|
||||
# convert to another type if arg is not a str
|
||||
if arg.annotation == int:
|
||||
_verified_args[arg.name] = int(args[arg.name])
|
||||
elif arg.annotation == float:
|
||||
_verified_args[arg.name] = float(args[arg.name])
|
||||
elif arg.annotation == bool:
|
||||
_verified_args[arg.name] = bool(args[arg.name])
|
||||
# TODO: more?
|
||||
else:
|
||||
_verified_args[arg.name] = args[arg.name]
|
||||
else:
|
||||
logging.info(
|
||||
f"{api_method.__name__}: Missing required argument: {arg.name}"
|
||||
)
|
||||
logging.info(
|
||||
f" [*] INVOKE ENDPOINT: {api_method.__name__}, ARGS: {', '.join(f'{k}={v}' for k, v in _verified_args.items())}"
|
||||
)
|
||||
_result: Any = await api_method(**_verified_args)
|
||||
logging.info(
|
||||
f" [*] ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}"
|
||||
)
|
||||
|
||||
if isinstance(_result, FileResponse):
|
||||
_json_result, _content_type = (
|
||||
await _convert_file_response(
|
||||
_result,
|
||||
message.connection,
|
||||
file_handler=StreamingFileHandler(),
|
||||
loop=loop,
|
||||
),
|
||||
"application/octet-stream",
|
||||
)
|
||||
else:
|
||||
_json_result = _result
|
||||
_content_type = "application/json"
|
||||
|
||||
_binary_result = (
|
||||
_json_result.encode("utf-8")
|
||||
if hasattr(_json_result, "encode")
|
||||
else (
|
||||
_json_result.body
|
||||
if hasattr(_json_result, "body")
|
||||
else (
|
||||
_json_result.read()
|
||||
if hasattr(_json_result, "read")
|
||||
else _json_result
|
||||
)
|
||||
)
|
||||
)
|
||||
logging.info(
|
||||
f" [x] ALL DONE in call_cleverthis_api id:{message.id}, resp code:{success_code}"
|
||||
)
|
||||
return DataResponseFactory.of(
|
||||
message.id,
|
||||
success_code,
|
||||
_content_type,
|
||||
_binary_result,
|
||||
message.trace_info,
|
||||
)
|
||||
except HTTPException as http_error:
|
||||
_content = str(http_error.detail)
|
||||
if not is_likely_json(_content):
|
||||
_content = '{"error": "' + _content + '"}'
|
||||
logging_error(f"Service REST endpoint invocation failed: {http_error}")
|
||||
return DataResponseFactory.of(
|
||||
message.id,
|
||||
http_error.status_code,
|
||||
"application/json",
|
||||
_content.encode("utf-8"),
|
||||
message.trace_info,
|
||||
)
|
||||
except Exception as error:
|
||||
logging_error(f"Service endpoint invocation failed: {error}")
|
||||
_content = str(error)
|
||||
if not is_likely_json(_content):
|
||||
_content = '{"error": "' + _content + '"}'
|
||||
return DataResponseFactory.of(
|
||||
message.id,
|
||||
500,
|
||||
"application/json",
|
||||
_content.encode("utf-8"),
|
||||
message.trace_info,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user