clevermicro/amq-adapter-python
14
0
Fork 0
Code Issues 16 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat/58 - improve Backpressure algorithm

Browse Source
This commit is contained in:
Stanislav Hejny
2025-07-15 16:40:02 +01:00
parent 7d4e358ce5
commit 7cd2960fda
6 changed files with 230 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/adapter/test_backpressure_handler.py
+61 -68
View File
@@ -13,9 +13,6 @@ from aio_pika import Exchange
from aio_pika.abc import AbstractRobustChannel
from amqp.adapter.backpressure_handler import (
RESOURCE_ACTIVE,
RESOURCE_OVERLOAD,
RESOURCE_UNSET,
BackpressureHandler,
RunningAverage,
ScaleRequestV1,
@@ -289,38 +286,38 @@ class TestBackpressureHandler:
assert handler.config == mock_config
assert handler.swarm_service_id == "clever-amqp-service"
assert handler.swarm_task_id == "python-adapter"
assert handler.parallel_workers == mock_config.backpressure.threshold_threads
assert handler.maximum_load == mock_config.backpressure.threshold_load
assert handler.average_cpu_usage is None
def test_increase_parallel_executions(self, mock_channel, mock_loop, mock_config):
def test_increase_current_load(self, mock_channel, mock_loop, mock_config):
"""
Test the increase_parallel_executions method.
Test the increase_current_load method.
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
initial_executions = handler.current_parallel_executions
handler.increase_parallel_executions()
assert handler.current_parallel_executions == initial_executions + 1
initial_load = handler.current_load
handler.increase_current_load()
assert handler.current_load == initial_load + 1
def test_decrease_parallel_executions(self, mock_channel, mock_loop, mock_config):
def test_decrease_current_load(self, mock_channel, mock_loop, mock_config):
"""
Test the decrease_parallel_executions method.
Test the decrease_current_load method.
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
handler.current_parallel_executions = 5
handler.decrease_parallel_executions()
assert handler.current_parallel_executions == 4
handler.current_load = 5
handler.decrease_current_load()
assert handler.current_load == 4
handler.current_parallel_executions = 0
handler.decrease_parallel_executions()
assert handler.current_parallel_executions == 0 # Should not go below 0
handler.current_load = 0
handler.decrease_current_load()
assert handler.current_load == 0 # Should not go below 0
def test_update_parallel_executions(self, mock_channel, mock_loop, mock_config):
def test_update_current_load(self, mock_channel, mock_loop, mock_config):
"""
Test the update_parallel_executions method.
Test the update_current_load method.
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
handler.update_parallel_executions(100)
assert handler.current_parallel_executions == 100
handler.update_current_load(100)
assert handler.current_load == 100
def test_update_last_data_message_time(self, mock_channel, mock_loop, mock_config):
"""
@@ -330,7 +327,7 @@ class TestBackpressureHandler:
with patch("time.time") as mock_time:
mock_time.return_value = 12345
handler.update_last_data_message_time()
assert handler.last_data_message_time == 12345
assert handler.last_backpressure_event_time == 12345
def test_start_backpressure_monitor(self, mock_channel, mock_loop, mock_config):
"""
@@ -345,19 +342,25 @@ class TestBackpressureHandler:
@pytest.mark.asyncio
async def test_check_overload_condition_no_overload(self, mock_channel, mock_loop, mock_config):
"""
Test the check_overload_condition method when there is no overload.
Test the check_overload_condition method when there is no overload and only short time elapsed from last message
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
handler.current_parallel_executions = 1
handler.parallel_workers = 5
handler.current_load = 1
handler.maximum_load = 5
handler.last_backpressure_event = ScalingRequestAlert.UPDATE
handler.publish_backpressure_request = AsyncMock()
handler.last_backpressure_event_time = 12340
with patch("time.time") as mock_time:
mock_time.return_value = 12345
with patch.object(handler, "handle_backpressure_overload_event") as mock_handle_overload:
with patch.object(
handler, "handle_backpressure_overload_event"
) as mock_handle_overload:
await handler.check_overload_condition()
mock_handle_overload.assert_not_called()
if 12345 - handler.last_backpressure_event_time > mock_config.backpressure.time_window:
if (
12345 - handler.last_backpressure_event_time
> mock_config.backpressure.time_window
):
handler.publish_backpressure_request.assert_called_once()
assert handler.last_backpressure_event == ScalingRequestAlert.UPDATE
assert handler.last_backpressure_event_time == 12345
@@ -370,7 +373,7 @@ class TestBackpressureHandler:
Test the check_overload_condition method when there is an overload.
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
handler.current_parallel_executions = 9 # Assuming parallel_workers is 10
handler.current_load = 9 # Assuming maximum_load is 10
handler.last_backpressure_event = ScalingRequestAlert.UPDATE
handler.publish_backpressure_request = AsyncMock()
with patch.object(handler, "handle_backpressure_overload_event") as mock_handle_overload:
@@ -390,7 +393,7 @@ class TestBackpressureHandler:
Test the check_overload_condition method when the last event was already an overload.
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
handler.current_parallel_executions = 10
handler.current_load = 10
handler.last_backpressure_event = ScalingRequestAlert.OVERLOAD
handler.publish_backpressure_request = AsyncMock()
with patch.object(handler, "handle_backpressure_overload_event") as mock_handle_overload:
@@ -418,7 +421,7 @@ class TestBackpressureHandler:
# 1. Simulate an overload condition:
handler.last_backpressure_event_time = 0 # Force event trigger
handler.last_data_message_time = 0
handler.last_backpressure_event_time = 0
handler.do_loop = 1
handler.config.backpressure.cpu_overload_duration = (
-1
@@ -432,7 +435,7 @@ class TestBackpressureHandler:
) # Mock the RunningAverage in IDLE state
handler.average_cpu_usage.is_filled = True # Together with 0 values ensures IDLE state
handler.last_backpressure_event_time = 0
handler.last_data_message_time = 0
handler.last_backpressure_event_time = 0
handler.do_loop = 1
handler.config.backpressure.idle_duration = (
-1
@@ -445,7 +448,7 @@ class TestBackpressureHandler:
handler._handle_backpressure_idle_event.call_count = 0
handler.average_cpu_usage = RunningAverage(100) # Mock the RunningAverage in pritine state
handler.last_backpressure_event_time = 0
handler.last_data_message_time = 0
handler.last_backpressure_event_time = 0
handler.do_loop = 1
await handler._backpressure_monitor()
assert (
@@ -465,7 +468,6 @@ class TestBackpressureHandler:
mock_time.return_value = 12345
await handler.handle_backpressure_overload_event()
assert handler.publish_backpressure_request.call_count == 1
assert handler.last_data_message_time == 12345
@pytest.mark.asyncio
async def test_handle_backpressure_idle_event(self, mock_channel, mock_loop, mock_config):
@@ -478,12 +480,11 @@ class TestBackpressureHandler:
mock_time.return_value = 12345
await handler._handle_backpressure_idle_event()
assert handler.publish_backpressure_request.call_count == 1
assert handler.last_data_message_time == 12345
@pytest.mark.asyncio
async def test_check_resource_availability_overload(self, mock_channel, mock_loop, mock_config):
async def test_update_backpressure_value_overload(self, mock_channel, mock_loop, mock_config):
"""
Test the check_resource_availability method when resource usage is in OVERLOAD state.
Test the update_backpressure_value method when resource usage is in OVERLOAD state.
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
handler.handle_backpressure_overload_event = AsyncMock()
@@ -492,17 +493,15 @@ class TestBackpressureHandler:
# Set up initial state
handler.last_backpressure_event = ScalingRequestAlert.UPDATE
handler.last_backpressure_event_time = 0
handler._resource_usage_state = RESOURCE_ACTIVE # Start with ACTIVE state
handler._resource_usage_changed = 0
handler.last_backpressure_event_time = 10 # time of last message
handler._resource_usage_changed = 10 # time of last change
with patch("time.time") as mock_time:
mock_time.return_value = 100
# Call with resource usage above OVERLOAD threshold (90% of maximum)
await handler.check_resource_availability(95, 100)
await handler.update_backpressure_value(95, 100)
# Debug output
print(f"Resource usage state: {handler._resource_usage_state}")
print(f"Last backpressure event: {handler.last_backpressure_event}")
print(f"Last backpressure event time: {handler.last_backpressure_event_time}")
print(f"Time window: {mock_config.backpressure.time_window}")
@@ -513,12 +512,11 @@ class TestBackpressureHandler:
handler.handle_backpressure_overload_event.assert_called_once()
assert handler.last_backpressure_event == ScalingRequestAlert.OVERLOAD
assert handler.last_backpressure_event_time == 100
assert handler._resource_usage_state == RESOURCE_OVERLOAD
@pytest.mark.asyncio
async def test_check_resource_availability_idle(self, mock_channel, mock_loop, mock_config):
async def test_update_backpressure_value_idle(self, mock_channel, mock_loop, mock_config):
"""
Test the check_resource_availability method when resource usage is in IDLE state.
Test the update_backpressure_value method when resource usage is in IDLE state.
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
handler._handle_backpressure_idle_event = AsyncMock()
@@ -528,17 +526,16 @@ class TestBackpressureHandler:
# Set up initial state
handler.last_backpressure_event = ScalingRequestAlert.UPDATE
handler.last_backpressure_event_time = 0
handler._resource_usage_state = RESOURCE_UNSET # Start with UNSET state
handler._resource_usage_changed = 0
with patch("time.time") as mock_time:
# First call to set state to IDLE
mock_time.return_value = 50
await handler.check_resource_availability(10, 100)
await handler.update_backpressure_value(0, 10)
# Second call after idle_duration has passed
mock_time.return_value = 50 + mock_config.backpressure.idle_duration + 1
await handler.check_resource_availability(10, 100)
await handler.update_backpressure_value(0, 10)
# Verify IDLE event was triggered
handler._handle_backpressure_idle_event.assert_called_once()
@@ -548,9 +545,9 @@ class TestBackpressureHandler:
)
@pytest.mark.asyncio
async def test_check_resource_availability_update(self, mock_channel, mock_loop, mock_config):
async def test_update_backpressure_value_update(self, mock_channel, mock_loop, mock_config):
"""
Test the check_resource_availability method when resource usage is in ACTIVE state.
Test the update_backpressure_value method when resource usage is in ACTIVE state.
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
handler.publish_backpressure_request = AsyncMock()
@@ -564,13 +561,12 @@ class TestBackpressureHandler:
with patch("time.time") as mock_time:
mock_time.return_value = 100
# Call with resource usage between thresholds
await handler.check_resource_availability(50, 100)
await handler.update_backpressure_value(50, 100)
# Verify UPDATE event was triggered
assert handler.publish_backpressure_request.call_count == 1
assert handler.last_backpressure_event == ScalingRequestAlert.UPDATE
assert handler.last_backpressure_event_time == 100
assert handler._resource_usage_state == RESOURCE_ACTIVE
@pytest.mark.asyncio
async def test_publish_backpressure_request_exchange_exists(
@@ -622,28 +618,28 @@ class TestBackpressureHandler:
Test the update_backpressure_value method.
"""
handler = BackpressureHandler(mock_channel, mock_loop, mock_config)
handler.update_parallel_executions = MagicMock()
handler.update_current_load = MagicMock()
handler.update_last_data_message_time = MagicMock()
handler.check_overload_condition = AsyncMock()
# Test with current value and same maximum
await handler.update_backpressure_value(5, handler.parallel_workers)
handler.update_parallel_executions.assert_called_once_with(5)
await handler.update_backpressure_value(5, handler.maximum_load)
handler.update_current_load.assert_called_once_with(5)
handler.update_last_data_message_time.assert_called_once()
handler.check_overload_condition.assert_called_once()
# Reset mocks
handler.update_parallel_executions.reset_mock()
handler.update_current_load.reset_mock()
handler.update_last_data_message_time.reset_mock()
handler.check_overload_condition.reset_mock()
# Test with current value and different maximum
original_workers = handler.parallel_workers
original_workers = handler.maximum_load
await handler.update_backpressure_value(3, original_workers + 5)
handler.update_parallel_executions.assert_called_once_with(3)
handler.update_current_load.assert_called_once_with(3)
handler.update_last_data_message_time.assert_called_once()
handler.check_overload_condition.assert_called_once()
assert handler.parallel_workers == original_workers + 5
assert handler.maximum_load == original_workers + 5
@pytest.mark.asyncio
async def test_check_overload_condition_idle(self, mock_channel, mock_loop, mock_config):
@@ -654,8 +650,8 @@ class TestBackpressureHandler:
handler._handle_backpressure_idle_event = AsyncMock()
# Set up initial state for IDLE condition
handler.current_parallel_executions = 0
handler.last_backpressure_event = ScalingRequestAlert.UPDATE
handler.current_load = 0
handler.last_backpressure_event = ScalingRequestAlert.IDLE
with patch("time.time") as mock_time:
# Set current time
@@ -663,11 +659,8 @@ class TestBackpressureHandler:
mock_time.return_value = current_time
# Set last data message time to be older than idle_duration
handler.last_data_message_time = (
current_time - mock_config.backpressure.idle_duration - 10
)
handler.last_backpressure_event_time = (
current_time - mock_config.backpressure.time_window - 10
current_time - mock_config.backpressure.idle_duration - 10
)
# Call the method
@@ -687,8 +680,8 @@ class TestBackpressureHandler:
handler.publish_backpressure_request = AsyncMock()
# Set up initial state for UPDATE condition
handler.current_parallel_executions = 2
handler.parallel_workers = 10
handler.current_load = 2
handler.maximum_load = 10
handler.last_backpressure_event = ScalingRequestAlert.IDLE # Coming from IDLE state
with patch("time.time") as mock_time:
@@ -720,8 +713,8 @@ class TestBackpressureHandler:
thread = handler.start_backpressure_monitor()
# Verify no thread was started
assert thread is None
assert not hasattr(handler, "thread") or handler.thread is None
assert thread is not None
assert hasattr(handler, "thread") or handler.thread is not None
def test_start_backpressure_monitor_enabled(self, mock_channel, mock_loop, mock_config):
"""