Converted AMQPAdapeter Java code to Python
This commit is contained in:
Stanislav Hejny
@@ -0,0 +1,84 @@
|
||||
import os
|
||||
import logging
|
||||
from functools import partial
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import (
|
||||
ConsoleSpanExporter,
|
||||
SimpleSpanProcessor
|
||||
)
|
||||
from opentelemetry.propagate import set_global_textmap
|
||||
from opentelemetry.propagators.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
from amqp.adapter.amq_message_factory import AMQMessageFactory
|
||||
from amqp.adapter.amq_route_factory import AMQRouteFactory
|
||||
from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
|
||||
from amqp.rabbitmq.rabbit_mq_consumer import RabbitMQConsumer
|
||||
from amqp.service.service_message_handler import AMQServiceMessageHandler
|
||||
|
||||
|
||||
class AMQService:
|
||||
CLEVER_AMQP_CLIENT_VERSION = os.getenv("CLEVER_AMQP_CLIENT_VERSION")
|
||||
MAX_WAIT = 300 # defaults to 5 minutes (300 seconds)
|
||||
|
||||
def __init__(self, amq_configuration, http_adapter):
|
||||
self.amq_configuration = amq_configuration
|
||||
self.http_adapter = http_adapter
|
||||
self.message_factory = AMQMessageFactory.get_instance(self.amq_configuration.amq_adapter().generator_id())
|
||||
self.rabbit_mq_consumer = RabbitMQConsumer(self.amq_configuration)
|
||||
self.amq_service_message_handler = AMQServiceMessageHandler(self.http_adapter, self.get_tracer())
|
||||
|
||||
def init(self):
|
||||
CleverMicroServiceMessageFactory.process_name(self.amq_configuration.amq_adapter().service_name())
|
||||
logging.info("********* AMQService.isRouter = %s ***********", self.amq_configuration.amq_adapter().is_router())
|
||||
logging.info("***********************************************")
|
||||
logging.info("********** AMQ Service *******************")
|
||||
logging.info("***********************************************")
|
||||
|
||||
self.setup_opentelemetry()
|
||||
self.register_routes()
|
||||
self.register_rpc_handler()
|
||||
self.rabbit_mq_consumer.request_routes()
|
||||
|
||||
def cleanup(self):
|
||||
logging.info(" [DBG] AMQService.cleanup() rmqc=%s", self.rabbit_mq_consumer)
|
||||
if self.rabbit_mq_consumer:
|
||||
self.rabbit_mq_consumer.cleanup()
|
||||
|
||||
def reinitialize_if_disconnected(self):
|
||||
if not self.rabbit_mq_consumer.channel or not self.rabbit_mq_consumer.channel.is_open:
|
||||
self.init()
|
||||
|
||||
def setup_opentelemetry(self):
|
||||
trace.set_tracer_provider(TracerProvider())
|
||||
trace.get_tracer_provider().add_span_processor(
|
||||
SimpleSpanProcessor(ConsoleSpanExporter())
|
||||
)
|
||||
set_global_textmap(TraceContextTextMapPropagator())
|
||||
self.tracer = trace.get_tracer("com.cleverthis")
|
||||
logging.info("OTEL %s %s %s", trace.get_logger_bridge(), trace.get_meter_provider(), trace.get_tracer_provider())
|
||||
logging.info("OTEL TRACER %s", trace.get_propagators())
|
||||
|
||||
def register_routes(self):
|
||||
route_mapping = self.amq_configuration.amq_adapter().route_mapping()
|
||||
if AMQRouteFactory.validate(route_mapping):
|
||||
try:
|
||||
self.rabbit_mq_consumer.register_routes()
|
||||
self.rabbit_mq_consumer.register_deliver_callback(
|
||||
partial(self.amq_service_message_handler.on_delivery_callback_http, self.rabbit_mq_consumer.channel)
|
||||
)
|
||||
self.rabbit_mq_consumer.register_rpc_handler(
|
||||
self.amq_service_message_handler.on_reply_consumer(self.rabbit_mq_consumer.channel)
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(e)
|
||||
elif route_mapping:
|
||||
logging.info(" [W] route [%s] failed validation, no routes were registered.", route_mapping)
|
||||
|
||||
def register_rpc_handler(self):
|
||||
try:
|
||||
self.rabbit_mq_consumer.register_rpc_handler(
|
||||
self.amq_service_message_handler.on_reply_consumer(self.rabbit_mq_consumer.get_channel())
|
||||
)
|
||||
except Exception as err:
|
||||
raise
|
||||
@@ -0,0 +1,143 @@
|
||||
import logging
|
||||
import time
|
||||
|
||||
from typing import Callable, Dict
|
||||
from concurrent.futures import Future
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.propagate import get_global_textmap, set_global_textmap
|
||||
from opentelemetry.propagators.tracecontext import TraceContextTextMapPropagator
|
||||
from opentelemetry.trace import Span, Tracer, TraceState
|
||||
|
||||
from amqp.adapter.amq_message_factory import AMQMessageFactory
|
||||
from amqp.model.model import AMQResponse
|
||||
from amqp.router.serializer import map_as_string
|
||||
|
||||
|
||||
class AMQServiceMessageHandler:
|
||||
def __init__(self, http_adapter, tracer):
|
||||
self.http_adapter = http_adapter
|
||||
self.tracer = tracer
|
||||
self.outstanding = {}
|
||||
|
||||
def on_delivery_callback_http(self, channel, consumerTag, delivery):
|
||||
start = time.time()
|
||||
deliveryTag = delivery.envelope.delivery_tag
|
||||
debug = f" [DBG] ######[{deliveryTag}] (AMQ MESSAGE), env: {delivery.envelope}, props: {delivery.properties}"
|
||||
logging.info(debug)
|
||||
|
||||
amq_message = AMQMessageFactory.from_bytes(delivery.body)
|
||||
logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, traceInfo=%s",
|
||||
deliveryTag, consumerTag, amq_message.id, map_as_string(amq_message.trace_info))
|
||||
|
||||
propagator = TraceContextTextMapPropagator.get_instance()
|
||||
context = propagator.extract(trace.get_current_context(), amq_message.trace_info , self.get_text_getter())
|
||||
parent_span = Span.from_context(context)
|
||||
context = context.with_span(parent_span)
|
||||
|
||||
with context.activate():
|
||||
logging.info(" [*] CTX=%s", context)
|
||||
current_span = self.tracer.start_span("AMQ-Adapter.Processing.Request", parent=parent_span)
|
||||
context_with_span = trace.set_span_in_context(current_span, context)
|
||||
logging.info(" [*] CTX2=%s", context_with_span)
|
||||
propagator.inject(context_with_span, amq_message.trace_info, self.set_text)
|
||||
logging.info(" [***] NEW TI=%s CTX=%s SCOPE=%s",
|
||||
map_as_string(amq_message.trace_info), context_with_span, context)
|
||||
|
||||
if amq_message.magic == AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST:
|
||||
future = self.http_adapter.handle(amq_message)
|
||||
future.add_done_callback(
|
||||
lambda f: self.on_http_response(delivery, deliveryTag, channel, f.result())
|
||||
)
|
||||
else:
|
||||
logging.error(" [E] Unknown MAGIC value: [%s], don't know how to interpret the message",
|
||||
amq_message.magic())
|
||||
|
||||
duration = time.time() - start
|
||||
if "___NACK___" in str(amq_message.body()):
|
||||
logging.info(" [x] ######[%s] Done, single NACK: %sms.", deliveryTag, duration)
|
||||
channel.basic_nack(deliveryTag, False, False)
|
||||
else:
|
||||
logging.info(" [x] ######[%s] Done, single ACK: %sms.", deliveryTag, duration)
|
||||
channel.basic_ack(deliveryTag, False)
|
||||
|
||||
def on_http_response(self, delivery, deliveryTag, channel, http_response):
|
||||
response = AMQResponse(
|
||||
delivery.properties.correlation_id,
|
||||
http_response.status_code,
|
||||
http_response.headers.get("Content-Type"),
|
||||
http_response.body,
|
||||
http_response.status_message,
|
||||
"",
|
||||
self.collect_trace_info(trace.get_current_context())
|
||||
)
|
||||
try:
|
||||
self.send_reply(delivery.properties.reply_to, deliveryTag, channel, response)
|
||||
except Exception as e:
|
||||
raise RuntimeError(e)
|
||||
|
||||
|
||||
class AMQServiceMessageHandler:
|
||||
def __init__(self, http_adapter, tracer):
|
||||
self.http_adapter = http_adapter
|
||||
self.tracer = tracer
|
||||
self.outstanding: Dict[str, 'ResponseHolder'] = {}
|
||||
|
||||
def collect_trace_info(self, context):
|
||||
map = {}
|
||||
TraceContextTextMapPropagator.get_instance().inject(
|
||||
context if context else trace.get_current_context(),
|
||||
map, self.set_text
|
||||
)
|
||||
return map
|
||||
|
||||
def send_reply(self, reply_to, delivery_tag, channel, response):
|
||||
if reply_to:
|
||||
logging.info(" [*] ######[%s] AMQ Message Reply, To=%s, msgId=%s",
|
||||
delivery_tag, reply_to, response.id())
|
||||
reply_props = pika.BasicProperties(
|
||||
correlation_id=response.id(),
|
||||
content_type=response.content_type()
|
||||
)
|
||||
channel.basic_publish(
|
||||
RouterBase.AMQ_REPLY_TO_EXCHANGE,
|
||||
reply_to,
|
||||
reply_props,
|
||||
AMQResponseFactory.serialize(response)
|
||||
)
|
||||
|
||||
def on_reply_consumer(self, channel):
|
||||
def handle_delivery(consumerTag, envelope, properties, body):
|
||||
reply_id = properties.correlation_id
|
||||
content_type = properties.content_type
|
||||
delivery_tag = envelope.delivery_tag
|
||||
debug = f" [DBG] ######[{delivery_tag}] (RPC REPLY) env: {envelope}, props: {properties}, BODY: {body.decode()}"
|
||||
logging.info(debug)
|
||||
print(f"OUTSTANDING RECV: {self.outstanding}, size={len(self.outstanding)}")
|
||||
|
||||
with self.outstanding.lock:
|
||||
reply = self.outstanding.pop(reply_id, None)
|
||||
if reply is None:
|
||||
logging.warn(" [W] ######[%s] IGNORING REPLY. No outstanding request for ID %s",
|
||||
delivery_tag, reply_id)
|
||||
else:
|
||||
result = AMQResponseFactory.from_bytes(body)
|
||||
reply.tracing_span().end()
|
||||
reply.response().set_result(Response.ok(result.body(), content_type).build())
|
||||
|
||||
channel.basic_ack(delivery_tag, False)
|
||||
|
||||
return handle_delivery
|
||||
|
||||
def register_response_holder(self, id, response_holder):
|
||||
self.outstanding[id] = response_holder
|
||||
logging.info(" [*] AMQMessage Sent, OUTSTANDING response holder registered: %s, size=%s",
|
||||
self.outstanding, len(self.outstanding))
|
||||
|
||||
def get_default_trace_state(self):
|
||||
return TraceState(entries={})
|
||||
|
||||
def get_text_getter(self):
|
||||
return lambda map, key: map.get(key)
|
||||
|
||||
def set_text(self, carrier, key, value):
|
||||
carrier[key] = value
|
||||
@@ -0,0 +1,89 @@
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.instrumentation.flask import FlaskInstrumentor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
||||
|
||||
from prometheus_client import start_http_server
|
||||
from opentelemetry.exporter.prometheus import PrometheusMetricReader
|
||||
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
|
||||
|
||||
import os
|
||||
|
||||
def initialize_trace(flask_app, name):
|
||||
# Resource can be required for some backends, e.g. Jaeger or Prometheus
|
||||
# If resource wouldn't be set - traces wouldn't appear in Jaeger.
|
||||
# The 'name' value is the name shown in Jaeger.
|
||||
resource = Resource(attributes={
|
||||
SERVICE_NAME: name if name else os.environ.get('APPLICATION_NAME', 'Python App')
|
||||
})
|
||||
#
|
||||
# 1. Tracing (Jaeger) configuration
|
||||
# ----------------------------------
|
||||
#
|
||||
# Initialize the tracer provider. This is abstract API call.
|
||||
# The actual implementation is given by the included dependency
|
||||
# The opentelemetry default is the OTLP trace provider.
|
||||
trace.set_tracer_provider(TracerProvider(resource=resource))
|
||||
#
|
||||
# Configure the OTLP exporter (or HTTP exporter the same way)
|
||||
# the recommended configuration is via environment variables. The mandatory ones:
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
|
||||
# OTEL_EXPORTER_OTLP_INSECURE: "true"
|
||||
# for more see here: https://opentelemetry-python.readthedocs.io/en/latest/exporter/otlp/otlp.html
|
||||
# alternatively these values can be provided as input values to OTLPSpanExporter() constructor.
|
||||
#
|
||||
otlp_exporter = OTLPSpanExporter()
|
||||
#
|
||||
# Add the exporters to the batch span processor
|
||||
span_processor = BatchSpanProcessor(otlp_exporter)
|
||||
trace.get_tracer_provider().add_span_processor(span_processor)
|
||||
|
||||
print(' * Instrumenting Flask application..... ')
|
||||
FlaskInstrumentor().instrument_app(flask_app)
|
||||
#
|
||||
# Python Flask is now automatically instrumented to export details of each REST endpoint call.
|
||||
# For Django based app, use 'opentelemetry-instrumentation-django' import instead. see here:
|
||||
# https://opentelemetry-python.readthedocs.io/en/latest/examples/django/README.html for example
|
||||
#
|
||||
# If the used library has no instrumentation support in opentelemetry.io, then the trace
|
||||
# can be started manually. In this code,
|
||||
# 'trace' should be global shared variable, initialized once per application (see tracing.py)
|
||||
# 'tracer' can be shared global or local variable, application can use multiple uniquely
|
||||
# named tracers, Jaeger will group the generated traces by the name of the tracer
|
||||
# that was used to start the Span. See this example code:
|
||||
#
|
||||
# tracer = trace.get_tracer(name)
|
||||
#
|
||||
# @app.route("/server_request")
|
||||
# def server_request():
|
||||
# with tracer.start_as_current_span(
|
||||
# "server_request",
|
||||
# context=extract(request.headers),
|
||||
# kind=trace.SpanKind.SERVER,
|
||||
# attributes=collect_request_attributes(request.environ),
|
||||
# ):
|
||||
# # do the work here, like print something and return response
|
||||
# print(request.args.get("param"))
|
||||
# return "served"
|
||||
#
|
||||
# Note:
|
||||
# the otlp code looks for HTTP header named 'traceparent' that, if present, should contain
|
||||
# the otlp parent span identification, and the span created here will be made a child
|
||||
# of that parent span, indicating visually the progression of requests via different services
|
||||
# for as long as the current span is provided in the traceparent header to next service.
|
||||
|
||||
|
||||
# 2. Metrics (Prometheus) configuration
|
||||
# -------------------------------------
|
||||
#
|
||||
# Start Prometheus client
|
||||
start_http_server(port=9464, addr="localhost")
|
||||
# Initialize PrometheusMetricReader which pulls metrics from the SDK
|
||||
# on-demand to respond to scrape requests (unlike Jaeger, which worls with push model - application
|
||||
# pushes traces when those become available to Jaeger's collector. Prometheus works with pull model
|
||||
# where it periodically polls pre-configured and/or discovered endpoints for metrics.
|
||||
reader = PrometheusMetricReader()
|
||||
provider = MeterProvider(resource=resource, metric_readers=[reader])
|
||||
metrics.set_meter_provider(provider)
|
||||
Reference in New Issue
Block a user