clevermicro/amq-adapter-python
14
0
Fork 0
Code Issues 16 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Add IDLE backpressure detection

Browse Source
This commit is contained in:
Stanislav Hejny
2025-04-06 01:24:51 +01:00
parent f7bffbe20d
commit fc110c7175
36 changed files with 904 additions and 1050 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amqp/adapter/cleverthis_service_adapter.py
+80 -89
View File
@@ -1,35 +1,33 @@
import asyncio
"""
The layer that interfaces with the CleverThis service that this Adapter is integrating with.
Methods here will invoke the actual CleverThis Service code, which are passed as Callable object.
"""
import json
import logging
import os
import re
import sys
import threading
import traceback
from aiohttp import ClientSession, ClientResponse
from fastapi import HTTPException
from typing import Dict, List, Tuple, Callable, Coroutine, AnyStr, Any
from urllib.parse import parse_qs
from xml.etree import ElementTree
from amqp.adapter.amq_response_factory import AMQResponseFactory
from amqp.service.multipart_parser import CleverMultiPartParser
from aiohttp import ClientSession, ClientResponse
from fastapi import HTTPException
from amqp.adapter import pydantic_serializer
from amqp.adapter.data_response_factory import DataResponseFactory
from amqp.config.amq_configuration import AMQConfiguration
from amqp.model.model import AMQMessage, AMQResponse
# Track the number of messages currently being processed
current_parallel_executions = 0
lock = threading.Lock()
parallel_workers = os.getenv("PARALLEL_WORKERS", default=10)
from amqp.model.model import DataMessage, DataResponse
from amqp.service.multipart_parser import CleverMultiPartParser
def parse_request_body(message: AMQMessage):
def parse_request_body(message: DataMessage):
"""
Parses the HTTP POST request body based on the MIME type.
Args:
message (AMQMessage): Contains raw request and MIME headers.
message (DataMessage): Contains raw request and MIME headers.
Returns:
dict: Parsed data as a dictionary.
@@ -82,53 +80,55 @@ def parse_request_body(message: AMQMessage):
raise ValueError(f"Unsupported Content-Type: {content_type}")
# ============= POST Helper function =============
async def call_cleverthis_post_put_api(
message: AMQMessage,
message: DataMessage,
args: Any,
api_method: Callable[[Any, Any], Coroutine],
success_code: int,
authenticated_user: Any
) -> AMQResponse:
) -> DataResponse:
"""
Handles POST and PUT requests, which typically contain possibly complex payload data, passed as a dictionary.
Invokes the provided API method with the given arguments and authenticated user,
and returns the response as AMQResponse.
:param message: AMQMessage containing the details of the call and the payload
and returns the response as DataResponse.
:param message: DataMessage containing the details of the call and the payload
:param args: map of the arguments to pass to the API method
:param api_method: Callable - the method to invoke. Last parameter is the authenticated user
:param success_code: HTTP code to be returned when operation suceeds
:param authenticated_user: Authenticated user. The actual class depends on the service
:return: AMQResponse class
:return: DataResponse class
"""
try:
coroutine_call: Coroutine = api_method(args, authenticated_user)
result: str = await coroutine_call
return AMQResponseFactory.of(
return DataResponseFactory.of(
message.id, success_code, message.headers.get("Content-Type", "application/json"),
result.encode('utf-8'), message.trace_info
)
except HTTPException as http_error:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {http_error}")
return AMQResponseFactory.of(message.id, http_error.status_code, "text/plain",
return DataResponseFactory.of(message.id, http_error.status_code, "text/plain",
str(http_error.detail).encode('utf-8'), message.trace_info)
except Exception as error:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {error}")
return AMQResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
return DataResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
# ============= GET Helper function =============
async def call_cleverthis_get_api(
message: AMQMessage,
message: DataMessage,
pattern_or_data: str | dict,
api_method: Callable[[Tuple[AnyStr | Any, ...], Any], Coroutine],
authenticated_user: Any
) -> AMQResponse:
) -> DataResponse:
"""
invokes provided Callable that is expected to be a GET API method, and returns the response as AMQResponse
:param message: AMQMessage containing the details of the call
:param pattern: If there are path variables, use this pattern to extract them
Handles GET requests, which do not contain a body, but may have path variables and extra values
in the query string.
invokes provided Callable that is expected to be a GET API method, and returns the response as DataResponse
:param message: DataMessage containing the details of the call
:param pattern_or_data: If there are path variables, use this pattern to extract them
:param api_method: Callable - the method to invoke. Last parameter is the authenticated user
:param authenticated_user: Authenticated user. The actual class depends on the service
:return:
@@ -138,25 +138,26 @@ async def call_cleverthis_get_api(
if isinstance(pattern_or_data, dict):
args = pattern_or_data
else:
match = re.search(pattern_or_data, message.path)
path, args = pydantic_serializer.parse_url_query(message.path)
match = re.search(pattern_or_data, path)
if match:
args = match.groups()
args.update(match.groupdict())
else:
_msg = f"Unexpected URL: {message.path}, expected pattern: {pattern_or_data}"
return AMQResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info)
return DataResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info)
crt: Coroutine = api_method(args, authenticated_user)
result: str = await crt
return AMQResponseFactory.of(message.id, 200, "text/plain", result.encode('utf-8'), message.trace_info)
return DataResponseFactory.of(message.id, 200, "text/plain", result.encode('utf-8'), message.trace_info)
except HTTPException as http_error:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {http_error}")
return AMQResponseFactory.of(message.id, http_error.status_code, "text/plain",
return DataResponseFactory.of(message.id, http_error.status_code, "text/plain",
str(http_error.detail).encode('utf-8'), message.trace_info)
except Exception as error:
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {error}")
return AMQResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
return DataResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
# ============= CleverThisServiceAdapter =============
@@ -164,58 +165,55 @@ class CleverThisServiceAdapter:
"""
A base class for CleverThis service adapter that handles the incoming AMQP messages and routes them to the
appropriate CleverThisService API endpoint.
IMPORTANT: This class is not intended to be used directly. It should be subclassed for each CleverThis service
and override the methods to handle the specific API calls.
"""
def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession = None):
self.amq_configuration = amq_configuration
self.session = session
self.service_host = self.amq_configuration.dispatch.service_host
self.service_port = self.amq_configuration.dispatch.service_port
self.require_authenticated_user = self.amq_configuration.amq_adapter.require_authenticated_user
def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
"""
Pull the value of the Content-Type header from the message headers.
:param message_headers: The headers of the message.
return: The content type of the message.
"""
for header, values in message_headers.items():
if header.lower() == 'content-type':
return values[0].split(';')[0]
return 'application/json'
async def get_current_user(self, token: str):
"""
To Be Overriden in subclass for given CleverThis Service, to return the current user based on the token,
in format appropriate for the service.
:param token: The token to be used for authentication.
:return: A user object or None if the user is not authenticated.
"""
return None
def sync_on_message(self, message: AMQMessage) -> AMQResponse:
"""
Synchronous version of on_message method. This is a wrapper around the async on_message method.
:param message: AMQMessage
:return: AMQResponse
"""
return asyncio.run(self.on_message(message))
async def on_message(self, message) -> AMQResponse:
async def on_message(self, message) -> DataResponse:
"""
Called when a message is received from the AMQP.
:param message:
:return: AMQP response class with operation status code and optional error details.
"""
DEBUG_NO_AUTH = 1
global current_parallel_executions
# Increment the counter for parallel executions
with lock:
current_parallel_executions += 1
logging.info(f"PARALLEL: Current Capacity [{current_parallel_executions}]")
if current_parallel_executions > parallel_workers - 2:
logging.warning("Warning: Capacity close to depleted!")
_auth: str = message.headers.get('Authorization', '')
_auth_user: Any | None = None
if _auth == '' and DEBUG_NO_AUTH == 1:
_auth = f"Bearer DEBUG_NO_AUTH"
amq_response: DataResponse | None = None
if 'Bearer' in _auth:
try:
token = _auth.split(' ')[1]
_auth_user = await self.get_current_user(token)
except Exception as error:
logging.error(f"Error getting user: {error}")
amq_response: AMQResponse | None = None
if _auth_user or DEBUG_NO_AUTH == 1:
if self.require_authenticated_user:
_auth: str = message.headers.get('Authorization', '')
if 'Bearer' in _auth:
try:
token = _auth.split(' ')[1]
_auth_user = await self.get_current_user(token)
except Exception as error:
logging.error(f"Error getting user: {error}")
if _auth_user or not self.require_authenticated_user:
method = message.method.lower()
if method == 'get':
amq_response = await self.get(message, _auth_user)
@@ -230,22 +228,21 @@ class CleverThisServiceAdapter:
else:
raise ValueError(f"Unexpected HTTP method: {message.method}")
else:
amq_response = AMQResponseFactory.of(message.id, 401, "text/plain", b"Unauthorized", message.trace_info)
with lock:
current_parallel_executions -= 1
amq_response = DataResponseFactory.of(message.id, 401, "text/plain", b"Unauthorized", message.trace_info)
return amq_response
# ==================================================================================================
# ===================== C l e v e r T h i s A P I m e t h o d s ============================
# ==================================================================================================
async def get(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
async def get(self, message: DataMessage, auth_user: Any) -> DataResponse:
"""
Handles the GET requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
It will attempt to call the REST endpoint directly using HTTP protocol.
:param message: message from the AMQP that contains the GET request
:param auth_user: user authenticated by the API Gateway, as UserSchem
:return: AMQResponse to be sent back to the API Gateway via AMQP
:return: DataResponse to be sent back to the API Gateway via AMQP
"""
# unmatched path - try to invoke the service directly on this path
@@ -254,19 +251,19 @@ class CleverThisServiceAdapter:
logging.info(f" [*] SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}")
if self.session:
_http_resp: ClientResponse = await self.session.get(url, headers=headers)
return AMQResponseFactory.of(message.id, _http_resp.status, _http_resp.content_type,
return DataResponseFactory.of(message.id, _http_resp.status, _http_resp.content_type,
await _http_resp.read(),
message.trace_info)
return AMQResponseFactory.of(message.id, 404, "text/plain",
return DataResponseFactory.of(message.id, 404, "text/plain",
str("Destination location does not exists").encode('utf-8'), message.trace_info)
async def put(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
async def put(self, message: DataMessage, auth_user: Any) -> DataResponse:
"""
Handles the PUT requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
It will attempt to call the REST endpoint directly using HTTP protocol.
:param message: message from the AMQP that contains the PUT request
:param auth_user: user authenticated by the API Gateway, as UserSchem
:return: AMQResponse to be sent back to the API Gateway via AMQP
:return: DataResponse to be sent back to the API Gateway via AMQP
"""
return await self.handle_possible_form(
message,
@@ -278,13 +275,13 @@ class CleverThisServiceAdapter:
auth_user
)
async def post(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
async def post(self, message: DataMessage, auth_user: Any) -> DataResponse:
"""
Handles the POST requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
It will attempt to call the REST endpoint directly using HTTP protocol.
:param message: message from the AMQP that contains the POST request, including body and headers
:param auth_user: user authenticated by the API Gateway, as UserSchem
:return: AMQResponse to be sent back to the API Gateway via AMQP
:return: DataResponse to be sent back to the API Gateway via AMQP
"""
return await self.handle_possible_form(
message,
@@ -296,32 +293,26 @@ class CleverThisServiceAdapter:
auth_user
)
async def handle_possible_form(self, message: AMQMessage, request_coroutine, auth_user: Any) -> AMQResponse:
async def handle_possible_form(self, message: DataMessage, request_coroutine, auth_user: Any) -> DataResponse:
args = parse_request_body(message)
return await request_coroutine
async def head(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
async def head(self, message: DataMessage, auth_user: Any) -> DataResponse:
"""
Handles the HEAD requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
It will attempt to call the REST endpoint directly using HTTP protocol.
:param message: message from the AMQP that contains the POST request, including body and headers
:param auth_user: user authenticated by the API Gateway, as UserSchem
:return: AMQResponse to be sent back to the API Gateway via AMQP
:return: DataResponse to be sent back to the API Gateway via AMQP
"""
raise NotImplementedError
async def delete(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
async def delete(self, message: DataMessage, auth_user: Any) -> DataResponse:
"""
Handles the DELETE requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
It will attempt to call the REST endpoint directly using HTTP protocol.
:param message: message from the AMQP that contains the POST request, including body and headers
:param auth_user: user authenticated by the API Gateway, as UserSchem
:return: AMQResponse to be sent back to the API Gateway via AMQP
:return: DataResponse to be sent back to the API Gateway via AMQP
"""
raise NotImplementedError
def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
for header, values in message_headers.items():
if header.lower() == 'content-type':
return values[0].split(';')[0]
return 'application/json'