From d8f133d1756e4ceff3d10c61f9b2bdbff548dfe0 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 14 Feb 2017 17:13:48 -0500 Subject: [PATCH 1/4] Add supervisor command to restart select services on reload event --- awx/main/tasks.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index d3d12b6259..458a4a022e 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -10,6 +10,7 @@ import json import logging import os import signal +import subprocess import pipes import re import shutil @@ -105,13 +106,38 @@ def _uwsgi_reload(): awxfifo.write(TRIGGER_CHAIN_RELOAD) -def _reset_celery_logging(): +def _reset_celery_thread_pool(): # Send signal to restart thread pool app = current_app._get_current_object() app.control.broadcast('pool_restart', arguments={'reload': True}, destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False) +def _supervisor_service_restart(): + ''' + example use pattern of supervisorctl: + # supervisorctl restart tower-processes:receiver tower-processes:factcacher + ''' + group_name = 'tower-processes' + args = ['supervisorctl'] + if settings.DEBUG is True: + args.extend(['-c', '/supervisor.conf']) + programs = "receiver,factcacher".split(",") + else: + programs = "awx-celeryd-beat,awx-callback-receiver,awx-fact-cache-receiver".split(",") + args.extend(['restart']) + args.extend(['{}:{}'.format(group_name, p) for p in programs]) + logger.debug('Issuing command to restart services, args={}'.format(args)) + subprocess.Popen(args) + + +def restart_local_services(): + logger.warn('Restarting services on this node in response to user action') + _uwsgi_reload() + _supervisor_service_restart() + _reset_celery_thread_pool() + + def _clear_cache_keys(set_of_keys): logger.debug('cache delete_many(%r)', set_of_keys) cache.delete_many(set_of_keys) @@ -125,8 +151,7 @@ def process_cache_changes(cache_keys): _clear_cache_keys(set_of_keys) for setting_key in set_of_keys: if setting_key.startswith('LOG_AGGREGATOR_'): - _uwsgi_reload() - _reset_celery_logging() + restart_local_services() break From 186b672e4f6f4cf864ffc361586d23a75a822a32 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 15 Feb 2017 12:07:30 -0500 Subject: [PATCH 2/4] move service definition into settings --- awx/main/tasks.py | 43 ++++++++++++++---------- awx/main/tests/unit/utils/test_reload.py | 13 +++++++ awx/settings/development.py | 12 +++++++ awx/settings/production.py | 12 +++++++ 4 files changed, 63 insertions(+), 17 deletions(-) create mode 100644 awx/main/tests/unit/utils/test_reload.py diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 458a4a022e..937a247b10 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -98,11 +98,7 @@ def _uwsgi_reload(): # http://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html#available-commands logger.warn('Initiating uWSGI chain reload of server') TRIGGER_CHAIN_RELOAD = 'c' - if settings.DEBUG: - uWSGI_FIFO_LOCATION = '/awxfifo' - else: - uWSGI_FIFO_LOCATION = '/var/lib/awx/awxfifo' - with open(uWSGI_FIFO_LOCATION, 'w') as awxfifo: + with open(settings.uWSGI_FIFO_LOCATION, 'w') as awxfifo: awxfifo.write(TRIGGER_CHAIN_RELOAD) @@ -113,29 +109,42 @@ def _reset_celery_thread_pool(): destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False) -def _supervisor_service_restart(): +def _supervisor_service_restart(service_internal_names): ''' + Service internal name options: + - beat - celery - callback - channels - uwsgi - daphne + - fact - nginx example use pattern of supervisorctl: # supervisorctl restart tower-processes:receiver tower-processes:factcacher ''' group_name = 'tower-processes' args = ['supervisorctl'] - if settings.DEBUG is True: + if settings.DEBUG: args.extend(['-c', '/supervisor.conf']) - programs = "receiver,factcacher".split(",") - else: - programs = "awx-celeryd-beat,awx-callback-receiver,awx-fact-cache-receiver".split(",") + programs = [] + name_translation_dict = settings.SERVICE_NAME_DICT + for n in service_internal_names: + if n in name_translation_dict: + programs.append('{}:{}'.format(group_name, name_translation_dict[n])) args.extend(['restart']) - args.extend(['{}:{}'.format(group_name, p) for p in programs]) + args.extend(programs) logger.debug('Issuing command to restart services, args={}'.format(args)) subprocess.Popen(args) -def restart_local_services(): - logger.warn('Restarting services on this node in response to user action') - _uwsgi_reload() - _supervisor_service_restart() - _reset_celery_thread_pool() +def restart_local_services(service_internal_names): + logger.warn('Restarting services {} on this node in response to user action'.format(service_internal_names)) + if 'uwsgi' in service_internal_names: + _uwsgi_reload() + service_internal_names.pop('uwsgi') + restart_celery = False + if 'celery' in service_internal_names: + restart_celery = True + service_internal_names.pop('celery') + _supervisor_service_restart(service_internal_names) + if restart_celery: + # Celery restarted last because this probably includes current process + _reset_celery_thread_pool() def _clear_cache_keys(set_of_keys): @@ -151,7 +160,7 @@ def process_cache_changes(cache_keys): _clear_cache_keys(set_of_keys) for setting_key in set_of_keys: if setting_key.startswith('LOG_AGGREGATOR_'): - restart_local_services() + restart_local_services(['uwsgi', 'celery', 'beat', 'callback', 'fact']) break diff --git a/awx/main/tests/unit/utils/test_reload.py b/awx/main/tests/unit/utils/test_reload.py new file mode 100644 index 0000000000..555f09eec5 --- /dev/null +++ b/awx/main/tests/unit/utils/test_reload.py @@ -0,0 +1,13 @@ +# from django.conf import LazySettings +import pytest + +# awx.main.utils.reload +from awx.main.main.tasks import _supervisor_service_restart, subprocess + + +def test_produce_supervisor_command(mocker): + with mocker.patch.object(subprocess, 'Popen'): + _supervisor_service_restart(['beat', 'callback', 'fact']) + subprocess.Popen.assert_called_once_with( + ['supervisorctl', 'restart', 'tower-processes:receiver', 'tower-processes:factcacher']) + diff --git a/awx/settings/development.py b/awx/settings/development.py index 1326c12814..23f79f7c60 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -112,3 +112,15 @@ except ImportError: CLUSTER_HOST_ID = socket.gethostname() CELERY_ROUTES['awx.main.tasks.cluster_node_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID} +# Supervisor service name dictionary used for programatic restart +SERVICE_NAME_DICT = { + "celery": "celeryd", + "callback": "receiver", + "runworker": "channels", + "uwsgi": "uwsgi", + "daphne": "daphne", + "fact": "factcacher", + "nginx": "nginx"} +# Used for sending commands in automatic restart +uWSGI_FIFO_LOCATION = '/awxfifo' + diff --git a/awx/settings/production.py b/awx/settings/production.py index f056a4ea31..92e5e6e81e 100644 --- a/awx/settings/production.py +++ b/awx/settings/production.py @@ -57,6 +57,18 @@ LOGGING['handlers']['fact_receiver']['filename'] = '/var/log/tower/fact_receiver LOGGING['handlers']['system_tracking_migrations']['filename'] = '/var/log/tower/tower_system_tracking_migrations.log' LOGGING['handlers']['rbac_migrations']['filename'] = '/var/log/tower/tower_rbac_migrations.log' +# Supervisor service name dictionary used for programatic restart +SERVICE_NAME_DICT = { + "beat": "awx-celeryd-beat", + "celery": "awx-celeryd", + "callback": "awx-callback-receiver", + "channels": "awx-channels-worker", + "uwsgi": "awx-uwsgi", + "daphne": "awx-daphne", + "fact": "awx-fact-cache-receiver"} +# Used for sending commands in automatic restart +uWSGI_FIFO_LOCATION = '/var/lib/awx/awxfifo' + # Store a snapshot of default settings at this point before loading any # customizable config files. DEFAULTS_SNAPSHOT = {} From 7e3a5fd2c26056a87c29eb4ef1b63fcbb854651a Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 15 Feb 2017 13:01:36 -0500 Subject: [PATCH 3/4] move reload functionality to its own file --- awx/main/tasks.py | 56 +------------------ awx/main/tests/unit/utils/test_reload.py | 38 ++++++++++--- awx/main/utils/reload.py | 68 ++++++++++++++++++++++++ awx/settings/development.py | 2 +- awx/settings/production.py | 2 +- 5 files changed, 102 insertions(+), 64 deletions(-) create mode 100644 awx/main/utils/reload.py diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 937a247b10..9791e15ccf 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -10,7 +10,6 @@ import json import logging import os import signal -import subprocess import pipes import re import shutil @@ -34,7 +33,6 @@ import pexpect # Celery from celery import Task, task from celery.signals import celeryd_init, worker_process_init -from celery import current_app # Django from django.conf import settings @@ -55,6 +53,7 @@ from awx.main.task_engine import TaskEnhancer from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url, check_proot_installed, build_proot_temp_dir, wrap_args_with_proot, get_system_task_capacity, OutputEventFilter, parse_yaml_or_json) +from awx.main.utils.reload import restart_local_services from awx.main.utils.handlers import configure_external_logger from awx.main.consumers import emit_channel_notification @@ -94,59 +93,6 @@ def task_set_logger_pre_run(*args, **kwargs): configure_external_logger(settings, async_flag=False, is_startup=False) -def _uwsgi_reload(): - # http://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html#available-commands - logger.warn('Initiating uWSGI chain reload of server') - TRIGGER_CHAIN_RELOAD = 'c' - with open(settings.uWSGI_FIFO_LOCATION, 'w') as awxfifo: - awxfifo.write(TRIGGER_CHAIN_RELOAD) - - -def _reset_celery_thread_pool(): - # Send signal to restart thread pool - app = current_app._get_current_object() - app.control.broadcast('pool_restart', arguments={'reload': True}, - destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False) - - -def _supervisor_service_restart(service_internal_names): - ''' - Service internal name options: - - beat - celery - callback - channels - uwsgi - daphne - - fact - nginx - example use pattern of supervisorctl: - # supervisorctl restart tower-processes:receiver tower-processes:factcacher - ''' - group_name = 'tower-processes' - args = ['supervisorctl'] - if settings.DEBUG: - args.extend(['-c', '/supervisor.conf']) - programs = [] - name_translation_dict = settings.SERVICE_NAME_DICT - for n in service_internal_names: - if n in name_translation_dict: - programs.append('{}:{}'.format(group_name, name_translation_dict[n])) - args.extend(['restart']) - args.extend(programs) - logger.debug('Issuing command to restart services, args={}'.format(args)) - subprocess.Popen(args) - - -def restart_local_services(service_internal_names): - logger.warn('Restarting services {} on this node in response to user action'.format(service_internal_names)) - if 'uwsgi' in service_internal_names: - _uwsgi_reload() - service_internal_names.pop('uwsgi') - restart_celery = False - if 'celery' in service_internal_names: - restart_celery = True - service_internal_names.pop('celery') - _supervisor_service_restart(service_internal_names) - if restart_celery: - # Celery restarted last because this probably includes current process - _reset_celery_thread_pool() - - def _clear_cache_keys(set_of_keys): logger.debug('cache delete_many(%r)', set_of_keys) cache.delete_many(set_of_keys) diff --git a/awx/main/tests/unit/utils/test_reload.py b/awx/main/tests/unit/utils/test_reload.py index 555f09eec5..3b8d66b56d 100644 --- a/awx/main/tests/unit/utils/test_reload.py +++ b/awx/main/tests/unit/utils/test_reload.py @@ -1,13 +1,37 @@ -# from django.conf import LazySettings -import pytest - # awx.main.utils.reload -from awx.main.main.tasks import _supervisor_service_restart, subprocess +from awx.main.utils import reload def test_produce_supervisor_command(mocker): - with mocker.patch.object(subprocess, 'Popen'): - _supervisor_service_restart(['beat', 'callback', 'fact']) - subprocess.Popen.assert_called_once_with( + with mocker.patch.object(reload.subprocess, 'Popen'): + reload._supervisor_service_restart(['beat', 'callback', 'fact']) + reload.subprocess.Popen.assert_called_once_with( ['supervisorctl', 'restart', 'tower-processes:receiver', 'tower-processes:factcacher']) + +def test_routing_of_service_restarts_works(mocker): + ''' + This tests that the parent restart method will call the appropriate + service restart methods, depending on which services are given in args + ''' + with mocker.patch.object(reload, '_uwsgi_reload'): + with mocker.patch.object(reload, '_reset_celery_thread_pool'): + with mocker.patch.object(reload, '_supervisor_service_restart'): + reload.restart_local_services(['uwsgi', 'celery', 'flower', 'daphne']) + reload._uwsgi_reload.assert_called_once_with() + reload._reset_celery_thread_pool.assert_called_once_with() + reload._supervisor_service_restart.assert_called_once_with(['flower', 'daphne']) + + +def test_routing_of_service_restarts_diables(mocker): + ''' + Test that methods are not called if not in the args + ''' + with mocker.patch.object(reload, '_uwsgi_reload'): + with mocker.patch.object(reload, '_reset_celery_thread_pool'): + with mocker.patch.object(reload, '_supervisor_service_restart'): + reload.restart_local_services(['flower']) + reload._uwsgi_reload.assert_not_called() + reload._reset_celery_thread_pool.assert_not_called() + reload._supervisor_service_restart.assert_called_once_with(['flower']) + diff --git a/awx/main/utils/reload.py b/awx/main/utils/reload.py new file mode 100644 index 0000000000..729a33a703 --- /dev/null +++ b/awx/main/utils/reload.py @@ -0,0 +1,68 @@ +# Copyright (c) 2017 Ansible Tower by Red Hat +# All Rights Reserved. + +# Python +import subprocess +import logging + +# Django +from django.conf import settings + +# Celery +from celery import current_app + +logger = logging.getLogger('awx.main.utils.reload') + + +def _uwsgi_reload(): + # http://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html#available-commands + logger.warn('Initiating uWSGI chain reload of server') + TRIGGER_CHAIN_RELOAD = 'c' + with open(settings.UWSGI_FIFO_LOCATION, 'w') as awxfifo: + awxfifo.write(TRIGGER_CHAIN_RELOAD) + + +def _reset_celery_thread_pool(): + # Send signal to restart thread pool + app = current_app._get_current_object() + app.control.broadcast('pool_restart', arguments={'reload': True}, + destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False) + + +def _supervisor_service_restart(service_internal_names): + ''' + Service internal name options: + - beat - celery - callback - channels - uwsgi - daphne + - fact - nginx + example use pattern of supervisorctl: + # supervisorctl restart tower-processes:receiver tower-processes:factcacher + ''' + group_name = 'tower-processes' + args = ['supervisorctl'] + if settings.DEBUG: + args.extend(['-c', '/supervisor.conf']) + programs = [] + name_translation_dict = settings.SERVICE_NAME_DICT + for n in service_internal_names: + if n in name_translation_dict: + programs.append('{}:{}'.format(group_name, name_translation_dict[n])) + args.extend(['restart']) + args.extend(programs) + logger.debug('Issuing command to restart services, args={}'.format(args)) + subprocess.Popen(args) + + +def restart_local_services(service_internal_names): + logger.warn('Restarting services {} on this node in response to user action'.format(service_internal_names)) + if 'uwsgi' in service_internal_names: + _uwsgi_reload() + service_internal_names.remove('uwsgi') + restart_celery = False + if 'celery' in service_internal_names: + restart_celery = True + service_internal_names.remove('celery') + _supervisor_service_restart(service_internal_names) + if restart_celery: + # Celery restarted last because this probably includes current process + _reset_celery_thread_pool() + diff --git a/awx/settings/development.py b/awx/settings/development.py index 23f79f7c60..0a0bc748f2 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -122,5 +122,5 @@ SERVICE_NAME_DICT = { "fact": "factcacher", "nginx": "nginx"} # Used for sending commands in automatic restart -uWSGI_FIFO_LOCATION = '/awxfifo' +UWSGI_FIFO_LOCATION = '/awxfifo' diff --git a/awx/settings/production.py b/awx/settings/production.py index 92e5e6e81e..19afcab9c9 100644 --- a/awx/settings/production.py +++ b/awx/settings/production.py @@ -67,7 +67,7 @@ SERVICE_NAME_DICT = { "daphne": "awx-daphne", "fact": "awx-fact-cache-receiver"} # Used for sending commands in automatic restart -uWSGI_FIFO_LOCATION = '/var/lib/awx/awxfifo' +UWSGI_FIFO_LOCATION = '/var/lib/awx/awxfifo' # Store a snapshot of default settings at this point before loading any # customizable config files. From 3023b4dfaa6289265c5e3f04c08a47540071ab2b Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 15 Feb 2017 14:53:01 -0500 Subject: [PATCH 4/4] switch to new nested with pattern from PR review --- awx/main/tests/unit/utils/test_reload.py | 29 ++++++++++++------------ 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/awx/main/tests/unit/utils/test_reload.py b/awx/main/tests/unit/utils/test_reload.py index 3b8d66b56d..d1f3291753 100644 --- a/awx/main/tests/unit/utils/test_reload.py +++ b/awx/main/tests/unit/utils/test_reload.py @@ -14,24 +14,25 @@ def test_routing_of_service_restarts_works(mocker): This tests that the parent restart method will call the appropriate service restart methods, depending on which services are given in args ''' - with mocker.patch.object(reload, '_uwsgi_reload'): - with mocker.patch.object(reload, '_reset_celery_thread_pool'): - with mocker.patch.object(reload, '_supervisor_service_restart'): - reload.restart_local_services(['uwsgi', 'celery', 'flower', 'daphne']) - reload._uwsgi_reload.assert_called_once_with() - reload._reset_celery_thread_pool.assert_called_once_with() - reload._supervisor_service_restart.assert_called_once_with(['flower', 'daphne']) + with mocker.patch.object(reload, '_uwsgi_reload'),\ + mocker.patch.object(reload, '_reset_celery_thread_pool'),\ + mocker.patch.object(reload, '_supervisor_service_restart'): + reload.restart_local_services(['uwsgi', 'celery', 'flower', 'daphne']) + reload._uwsgi_reload.assert_called_once_with() + reload._reset_celery_thread_pool.assert_called_once_with() + reload._supervisor_service_restart.assert_called_once_with(['flower', 'daphne']) + def test_routing_of_service_restarts_diables(mocker): ''' Test that methods are not called if not in the args ''' - with mocker.patch.object(reload, '_uwsgi_reload'): - with mocker.patch.object(reload, '_reset_celery_thread_pool'): - with mocker.patch.object(reload, '_supervisor_service_restart'): - reload.restart_local_services(['flower']) - reload._uwsgi_reload.assert_not_called() - reload._reset_celery_thread_pool.assert_not_called() - reload._supervisor_service_restart.assert_called_once_with(['flower']) + with mocker.patch.object(reload, '_uwsgi_reload'),\ + mocker.patch.object(reload, '_reset_celery_thread_pool'),\ + mocker.patch.object(reload, '_supervisor_service_restart'): + reload.restart_local_services(['flower']) + reload._uwsgi_reload.assert_not_called() + reload._reset_celery_thread_pool.assert_not_called() + reload._supervisor_service_restart.assert_called_once_with(['flower'])