"""
methods to create, serialize and deserialize DataMessage objects.
"""
import base64
import json
import re
from datetime import datetime, timezone
from io import BytesIO
from typing import Any, Dict, List
import amqp.adapter.file_handler
from amqp.adapter.file_handler import FileHandler, StreamingFileHandler
from amqp.adapter.pydantic_serializer import python_type_to_json
from amqp.adapter.serializer import (
map_as_string,
map_with_list_as_string,
parse_map_list_string,
parse_map_string,
)
from amqp.model.model import DataMessage, DataMessageMagicByte
from amqp.model.snowflake_id import SnowflakeId
from amqp.router.utils import sanitize_as_rabbitmq_name
[docs]
class DataMessageFactory:
SNOWFLAKE_ID_BITS = 80
SEQUENCE_BITS = 12
MACHINE_ID_BITS = 24
TIMESTAMP_BITS = SNOWFLAKE_ID_BITS - SEQUENCE_BITS - MACHINE_ID_BITS
MACHINE_ID_MASK = (1 << MACHINE_ID_BITS) - 1
SEQUENCE_MASK = (1 << SEQUENCE_BITS) - 1
_instance = None
snowflake_epoch = datetime(2024, 10, 1, 0, 0, tzinfo=timezone.utc).timestamp()
[docs]
def __init__(self, machine_id: int):
self.machine_id = (machine_id & self.MACHINE_ID_MASK) << self.SEQUENCE_BITS
self.snowflake_sequence = 0
self.snowflake_last_timestamp = 0
[docs]
@classmethod
def get_instance(cls, machine_id: int):
"""
return reusable factory instance
:param machine_id: a generator ID, should be unique for each component to prevent duplicate IDs
:return: factory instance
"""
if cls._instance is None:
cls._instance = DataMessageFactory(machine_id)
return cls._instance
[docs]
def generate_snowflake_id(self) -> SnowflakeId:
"""
Generate an unique ID following the SnowflakeID structure, see https://en.wikipedia.org/wiki/Snowflake_ID
:return: 80-bit wide ID
"""
timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
if timestamp < self.snowflake_last_timestamp:
raise RuntimeError("Clock moved backwards. Refusing to generate id")
if timestamp == self.snowflake_last_timestamp:
self.snowflake_sequence = (self.snowflake_sequence + 1) & self.SEQUENCE_MASK
if self.snowflake_sequence == 0:
while timestamp == self.snowflake_last_timestamp:
timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
self.snowflake_last_timestamp = timestamp
else:
self.snowflake_sequence = 0
self.snowflake_last_timestamp = timestamp
time = timestamp - 1000 * int(DataMessageFactory.snowflake_epoch)
return SnowflakeId(
time >> (64 - self.SEQUENCE_BITS - self.MACHINE_ID_BITS),
(
(time << (self.SEQUENCE_BITS + self.MACHINE_ID_BITS))
| self.machine_id
| self.snowflake_sequence
)
& 0xFFFFFFFFFFFFFFFF,
)
[docs]
def create_request_message(
self,
method: str,
domain: str,
path: str,
headers: Dict[str, List[str]],
message,
) -> DataMessage:
"""
Encapsulate / hide the system level inputs, create the AMQ message from business relevant input
:param method: a method requested (e.g. HTTP method like GET, POST in case of REST request)
:param domain: service host name
:param path: path at the service
:param headers: HTTP headers
:param message: payload
:return: DataMessage object
"""
serializable_headers = headers.copy() if headers else {}
payload = base64.b64encode(message.encode("utf-8") if isinstance(message, str) else message)
trace_info = {}
return DataMessage(
DataMessageMagicByte.HTTP_REQUEST.value,
self.generate_snowflake_id(),
method,
domain,
path,
serializable_headers,
trace_info,
payload,
)
[docs]
def safe_rsplit(self, fq_method_name, default="_"):
"""
Safely split a fully qualified method name into package, class, and method
Args:
fq_method_name (str): Fully qualified method name
default (str, optional): Default value for missing tokens. Defaults to ''.
Returns:
tuple: (package, class_name, method)
"""
if not fq_method_name:
return default, default, default
parts = fq_method_name.rsplit(".", 2)
# Pad with default values if insufficient tokens
while len(parts) < 3:
parts.insert(0, default)
return tuple(parts)
[docs]
def create_rpc_message(
self,
fq_method_name: str,
swarm_id: str,
args: Dict[str, Any] = None,
) -> DataMessage:
"""
Create a RPC message with the given parameters.
:param fq_method_name: fully qualified method name (e.g. 'com.example.ServiceClass.method_name')
:param args: arguments whose serialized form will form the request body
:return: DataMessage object
"""
package, class_name, method = self.safe_rsplit(fq_method_name)
headers = {"Content-Type": ["application/octet-stream"], "Return-Address": [swarm_id]}
trace_info = {}
body = python_type_to_json(args).encode(encoding="utf-8") if args else b""
print(
f"Creating RPC message for method: {fq_method_name}, args: {args} body: {body.decode('utf-8') if body else 'None'}"
)
return DataMessage(
DataMessageMagicByte.RPC_REQUEST.value,
self.generate_snowflake_id(),
package,
class_name,
method,
headers,
trace_info,
body,
)
[docs]
@staticmethod
def amq_message_routing_key(message: DataMessage) -> str:
"""
Constructs the message specific routing key.
:param message: DataMessage input
:return: routing key (as string) compliant to RabbitMQ allowed character set.
"""
return sanitize_as_rabbitmq_name(f"{message.domain()}.{message.path()}")
[docs]
@staticmethod
def get_timestamp_from_id(_id: SnowflakeId) -> int:
"""
Helper method - retrieve the timestamp from the ID
:param _id: SnowflakeID (includes encoded timestamp)
:return: the message timestamp
"""
bits = _id.get_bits()
low = bits[0] >> (DataMessageFactory.SEQUENCE_BITS + DataMessageFactory.MACHINE_ID_BITS)
high = bits[1] << (
64 - DataMessageFactory.SEQUENCE_BITS - DataMessageFactory.MACHINE_ID_BITS
)
return (high | low) + DataMessageFactory.snowflake_epoch * 1000
[docs]
@staticmethod
def to_string(message: DataMessage) -> str:
"""
Human-readable representation.
:param message: DataMessage input
:return: as string
"""
return (
f"DataMessage{{"
f"id={message.id()}, "
f"method='{message.method()}', "
f"domain='{message.domain()}', "
f"path='{message.path()}', "
f"headers={{{map_with_list_as_string(message.headers())}}}, "
f"traceInfo={{{map_as_string(message.trace_info())}}}, "
f"base64body={message.base64body().decode()}"
f"}}"
)
[docs]
@staticmethod
def serialize(message: DataMessage) -> bytes:
"""
Converts DataMessage to byte array
:param message: message to serialize
:return: messages as bytes
"""
id_bytes = str(message.id()).encode()
prolog = (
f"|m={message.method()}|d={message.domain()}|p={message.path()}|h={{{map_with_list_as_string(message.headers())}}}|t={{{map_as_string(message.trace_info())}}}|b="
).encode("utf-8")
header_length = 1 + len(id_bytes) + len(prolog)
prolog_len = header_length.to_bytes(2, byteorder="big")
msg_length = 2 + header_length + len(message.base64body())
baos = BytesIO(bytearray(msg_length))
baos.write(prolog_len)
baos.write(message.magic().encode("utf-8"))
baos.write(id_bytes)
baos.write(prolog)
baos.write(message.base64body())
return baos.getvalue()
[docs]
@staticmethod
def from_bytes(input_bytes: bytes) -> DataMessage:
"""
Builds the DataMessage from its serialized (byte array) form.
:param input_bytes: serialized message
:return: DataMessage instance
"""
prolog_len = (input_bytes[0] << 8) + input_bytes[1]
metadata = input_bytes[2 : prolog_len + 2].decode()
pattern = re.compile(
r"([AR])"
r"([0-9A-Fa-f]+)\|"
r"m=([^|]*)\|"
r"d=([^|]*)\|"
r"p=([^|]*)\|"
r"h=(\{[^}]*\})\|"
r"t=(\{[^}]*\})\|"
r"b="
)
match = pattern.match(metadata)
if not match:
raise ValueError(
f"DataMessage format does not match expected format. "
f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
)
id_str = match.group(2)
method = match.group(3)
domain = match.group(4)
path = match.group(5)
headers_str = match.group(6)
trace_info_str = match.group(7)
body_b64 = input_bytes[prolog_len + 2 :]
headers = parse_map_list_string(headers_str)
trace_info = parse_map_string(trace_info_str)
return DataMessage(
match.group(1), # Magic byte (R for RPC, A for HTTP)
SnowflakeId.from_hex(id_str),
method,
domain,
path,
headers,
trace_info,
body_b64,
)
[docs]
@staticmethod
async def from_stream(
stream: bytes,
rabbitmq_url: str,
loop,
file_handler: FileHandler = StreamingFileHandler(),
) -> DataMessage | None:
"""
Builds the DataMessage from its serialized (byte array) form.
:param stream: serialized message
:param rabbitmq_url: RabbitMQ connection URL
:param loop: event loop
:param file_handler: file handler for downloading the message
:return: DataMessage instance
"""
# async implementation
_message_data = json.loads(stream.decode())
_req_info = _message_data.get("req-info", "")
if _req_info:
(body, eof) = await file_handler.download_buffer(
loop=loop, rabbitmq_url=rabbitmq_url, queue_name=_req_info
)
if eof == amqp.adapter.file_handler.END_OF_FILE:
return DataMessageFactory.from_bytes(body)
return None