clevermicro/amq-adapter-python
14
0
Fork 0
Code Issues 16 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Fix the global unique ID generator recursive constructor call

Browse Source
This commit is contained in:
Stanislav Hejny
2025-07-12 22:59:21 +01:00
parent dd590a11b5
commit 0e294b223d
8 changed files with 122 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amqp/adapter/consul_global_id_generator.py
+34 -40
View File
@@ -1,51 +1,46 @@
import os
import logging
import os
import random
import socket
import time
from typing import Optional
import consul
from amqp.config.amq_configuration import AMQConfiguration
import consul_kv
from amqp.config.amq_configuration import AMQAdapter
logger = logging.getLogger(__name__)
# Default configuration values for Adapter
DEFAULT_CONSUL_HOST = 'localhost'
DEFAULT_CONSUL_PORT = 8500
DEFAULT_CONSUL_COUNTER_KEY = 'adapter/ids/counter'
DEFAULT_MAX_RETRIES = 5
DEFAULT_RETRY_DELAY_SECONDS = 0.1
def get_config_values():
def get_config_values(config: AMQAdapter) -> tuple:
"""
Get configuration values from AMQConfiguration or environment variables.
Returns:
tuple: (consul_host, consul_port, consul_counter_key, max_retries, retry_delay_seconds)
"""
try:
config = AMQConfiguration("application.properties")
consul_host = os.environ.get('CONSUL_HOST', DEFAULT_CONSUL_HOST)
consul_port = int(os.environ.get('CONSUL_PORT', DEFAULT_CONSUL_PORT))
consul_counter_key = os.environ.get('CONSUL_COUNTER_KEY', DEFAULT_CONSUL_COUNTER_KEY)
max_retries = int(os.environ.get('CONSUL_MAX_RETRIES', DEFAULT_MAX_RETRIES))
retry_delay_seconds = float(os.environ.get('CONSUL_RETRY_DELAY_SECONDS', DEFAULT_RETRY_DELAY_SECONDS))
consul_host = config.consul_host
consul_port = config.consul_port
consul_counter_key = config.consul_counter_key
max_retries = config.consul_max_retries
retry_delay_seconds = config.consul_initial_retry_delay
return consul_host, consul_port, consul_counter_key, max_retries, retry_delay_seconds
except Exception as e:
logger.warning(f"Failed to load configuration: {str(e)}. Using default values.")
return (
os.environ.get('CONSUL_HOST', DEFAULT_CONSUL_HOST),
int(os.environ.get('CONSUL_PORT', DEFAULT_CONSUL_PORT)),
os.environ.get('CONSUL_COUNTER_KEY', DEFAULT_CONSUL_COUNTER_KEY),
int(os.environ.get('CONSUL_MAX_RETRIES', DEFAULT_MAX_RETRIES)),
float(os.environ.get('CONSUL_RETRY_DELAY_SECONDS', DEFAULT_RETRY_DELAY_SECONDS))
os.environ.get("CONSUL_HOST", consul_kv.settings.DEFAULT_ENDPOINT.split(":")[0]),
int(os.environ.get("CONSUL_PORT", consul_kv.settings.DEFAULT_ENDPOINT.split(":")[1])),
os.environ.get("CONSUL_COUNTER_KEY", "service/ids/counter"),
int(os.environ.get("CONSUL_MAX_RETRIES", 5)),
float(os.environ.get("CONSUL_RETRY_DELAY_SECONDS", 0.2)),
)
def generate_fallback_id() -> int:
"""
Generate a fallback ID when Consul is not available.
Uses a combination of timestamp, hostname hash, and random number.
Returns:
int: A reasonably unique integer ID
"""
@@ -57,44 +52,43 @@ def generate_fallback_id() -> int:
logger.warning(f"Using fallback ID generation method: {unique_id}")
return unique_id
def get_unique_instance_id() -> int:
def get_unique_instance_id(config: AMQAdapter) -> int:
"""
Get a globally unique ID from Consul.
Uses Consul's atomic Compare-And-Set operations to safely increment a counter.
Falls back to a local generation method if Consul is unavailable.
Returns:
int: A globally unique integer ID
"""
consul_host, consul_port, consul_counter_key, max_retries, retry_delay_seconds = get_config_values()
consul_host, consul_port, consul_counter_key, max_retries, retry_delay_seconds = (
get_config_values(config)
)
try:
c = consul.Consul(host=consul_host, port=consul_port)
index, data = c.kv.get(consul_counter_key)
c = consul_kv.Connection(endpoint=f"{consul_host}:{consul_port}", timeout=5)
index, data = c.get(consul_counter_key)
if data is None:
logger.info(f"Initializing Consul counter at {consul_counter_key}")
if c.kv.put(consul_counter_key, "1"):
if c.put(consul_counter_key, "1", cas=1):
return 1
else:
logger.error("Failed to initialize Consul counter")
return generate_fallback_id()
current_value = int(data['Value'].decode('utf-8'))
current_value = int(data["Value"].decode("utf-8"))
new_value = current_value + 1
for attempt in range(max_retries):
success = c.kv.put(
consul_counter_key,
str(new_value),
cas=data['ModifyIndex']
)
success = c.put(consul_counter_key, str(new_value), cas=data["ModifyIndex"])
if success:
logger.debug(f"Successfully obtained unique ID: {new_value}")
return new_value
logger.debug(f"CAS update failed on attempt {attempt+1}, retrying...")
time.sleep(retry_delay_seconds)
index, data = c.kv.get(consul_counter_key)
logger.debug(f"CAS update failed on attempt {attempt + 1}, retrying...")
time.sleep(retry_delay_seconds * (1 << attempt)) # Exponential backoff
index, data = c.get(consul_counter_key)
if data is None:
logger.error("Counter disappeared during update")
return generate_fallback_id()
current_value = int(data['Value'].decode('utf-8'))
current_value = int(data["Value"].decode("utf-8"))
new_value = current_value + 1
logger.error(f"Failed to update counter after {max_retries} attempts")
return generate_fallback_id()