External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-24 12:25:01 -02:30
Code Issues Packages Projects Releases Wiki Activity

broadcast queues get a per-node stable queue name

Browse Source 
* Using Kombu's default Broadcast() constructor requires only 1
parameter. That parameter defines the exchange name and the queue name
is randomly generated per-node.
* This caused problems if/when celery enters an infinite restart loop
because too many rabbit queues get created and rabbit OOM's
(gracefully).
* To remedy this we tell Broadcast the queue name to use, which is
derived from some constant + the node name so that the per-node queue
name is stable.
This commit is contained in:
chris meyers
2018-04-30 15:18:05 -04:00
parent a009d21edc
commit 648d9165ff
4 changed files with 40 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
awx/main/tests/unit/utils/test_ha.py
View File

@@ -6,6 +6,7 @@
# python
import pytest
import mock
from contextlib import nested
# AWX
from awx.main.utils.ha import (
@@ -47,22 +48,26 @@ class TestAddRemoveCeleryWorkerQueues():
app.control.cancel_consumer = mocker.MagicMock()
return app
@pytest.mark.parametrize("static_queues,_worker_queues,groups,hostname,added_expected,removed_expected", [
(['east', 'west'], ['east', 'west', 'east-1'], [], 'east-1', [], []),
([], ['east', 'west', 'east-1'], ['east', 'west'], 'east-1', [], []),
([], ['east', 'west'], ['east', 'west'], 'east-1', ['east-1'], []),
([], [], ['east', 'west'], 'east-1', ['east', 'west', 'east-1'], []),
([], ['china', 'russia'], ['east', 'west'], 'east-1', ['east', 'west', 'east-1'], ['china', 'russia']),
@pytest.mark.parametrize("broadcast_queues,static_queues,_worker_queues,groups,hostname,added_expected,removed_expected", [
(['tower_broadcast_all'], ['east', 'west'], ['east', 'west', 'east-1'], [], 'east-1', ['tower_broadcast_all_east-1'], []),
([], [], ['east', 'west', 'east-1'], ['east', 'west'], 'east-1', [], []),
([], [], ['east', 'west'], ['east', 'west'], 'east-1', ['east-1'], []),
([], [], [], ['east', 'west'], 'east-1', ['east', 'west', 'east-1'], []),
([], [], ['china', 'russia'], ['east', 'west'], 'east-1', ['east', 'west', 'east-1'], ['china', 'russia']),
])
def test__add_remove_celery_worker_queues_noop(self, mock_app,
instance_generator,
worker_queues_generator,
static_queues, _worker_queues,
instance_generator,
worker_queues_generator,
broadcast_queues,
static_queues, _worker_queues,
groups, hostname,
added_expected, removed_expected):
instance = instance_generator(groups=groups, hostname=hostname)
worker_queues = worker_queues_generator(_worker_queues)
with mock.patch('awx.main.utils.ha.settings.AWX_CELERY_QUEUES_STATIC', static_queues):
with nested(
mock.patch('awx.main.utils.ha.settings.AWX_CELERY_QUEUES_STATIC', static_queues),
mock.patch('awx.main.utils.ha.settings.AWX_CELERY_BCAST_QUEUES_STATIC', broadcast_queues),
mock.patch('awx.main.utils.ha.settings.CLUSTER_HOST_ID', hostname)):
(added_queues, removed_queues) = _add_remove_celery_worker_queues(mock_app, [instance], worker_queues, hostname)
assert set(added_queues) == set(added_expected)
assert set(removed_queues) == set(removed_expected)
@@ -71,11 +76,11 @@ class TestAddRemoveCeleryWorkerQueues():
class TestUpdateCeleryWorkerRoutes():
@pytest.mark.parametrize("is_controller,expected_routes", [
(False, {
(False, {
'awx.main.tasks.cluster_node_heartbeat': {'queue': 'east-1', 'routing_key': 'east-1'},
'awx.main.tasks.purge_old_stdout_files': {'queue': 'east-1', 'routing_key': 'east-1'}
}),
(True, {
(True, {
'awx.main.tasks.cluster_node_heartbeat': {'queue': 'east-1', 'routing_key': 'east-1'},
'awx.main.tasks.purge_old_stdout_files': {'queue': 'east-1', 'routing_key': 'east-1'},
'awx.main.tasks.awx_isolated_heartbeat': {'queue': 'east-1', 'routing_key': 'east-1'},