mirror of
https://github.com/ansible/awx.git
synced 2026-01-17 12:41:19 -03:30
Fixing up isolated node execution after cluster changes
* Rework queue detection to include control groups and isolated instances * Fix up development tooling around isolated nodes * Update unit tests
This commit is contained in:
Matthew Jones
parent
0268d575f8
commit
925d9efecf
@ -206,20 +206,22 @@ def handle_ha_toplogy_changes(self):
|
||||
logger.debug("Reconfigure celeryd queues task on host {}".format(self.request.hostname))
|
||||
awx_app = Celery('awx')
|
||||
awx_app.config_from_object('django.conf:settings', namespace='CELERY')
|
||||
(instance, removed_queues, added_queues) = register_celery_worker_queues(awx_app, self.request.hostname)
|
||||
logger.info("Workers on tower node '{}' removed from queues {} and added to queues {}"
|
||||
.format(instance.hostname, removed_queues, added_queues))
|
||||
updated_routes = update_celery_worker_routes(instance, settings)
|
||||
logger.info("Worker on tower node '{}' updated celery routes {} all routes are now {}"
|
||||
.format(instance.hostname, updated_routes, self.app.conf.CELERY_TASK_ROUTES))
|
||||
instances, removed_queues, added_queues = register_celery_worker_queues(awx_app, self.request.hostname)
|
||||
for instance in instances:
|
||||
logger.info("Workers on tower node '{}' removed from queues {} and added to queues {}"
|
||||
.format(instance.hostname, removed_queues, added_queues))
|
||||
updated_routes = update_celery_worker_routes(instance, settings)
|
||||
logger.info("Worker on tower node '{}' updated celery routes {} all routes are now {}"
|
||||
.format(instance.hostname, updated_routes, self.app.conf.CELERY_TASK_ROUTES))
|
||||
|
||||
|
||||
@worker_ready.connect
|
||||
def handle_ha_toplogy_worker_ready(sender, **kwargs):
|
||||
logger.debug("Configure celeryd queues task on host {}".format(sender.hostname))
|
||||
(instance, removed_queues, added_queues) = register_celery_worker_queues(sender.app, sender.hostname)
|
||||
logger.info("Workers on tower node '{}' unsubscribed from queues {} and subscribed to queues {}"
|
||||
.format(instance.hostname, removed_queues, added_queues))
|
||||
instances, removed_queues, added_queues = register_celery_worker_queues(sender.app, sender.hostname)
|
||||
for instance in instances:
|
||||
logger.info("Workers on tower node '{}' unsubscribed from queues {} and subscribed to queues {}"
|
||||
.format(instance.hostname, removed_queues, added_queues))
|
||||
|
||||
|
||||
@celeryd_init.connect
|
||||
|
||||
@ -136,7 +136,8 @@ class TestIsolatedManagementTask:
|
||||
|
||||
# Upgrade was completed, health check playbook now reports matching
|
||||
# version, make sure capacity is set.
|
||||
update_capacity(old_version, {'version': '5.0.0-things', 'capacity':103}, '5.0.0-stuff')
|
||||
update_capacity(old_version, {'version': '5.0.0-things',
|
||||
'capacity_cpu':103, 'capacity_mem':103}, '5.0.0-stuff')
|
||||
assert old_version.capacity == 103
|
||||
|
||||
def test_takes_action(self, control_instance, needs_updating):
|
||||
|
||||
@ -64,7 +64,7 @@ class TestAddRemoveCeleryWorkerQueues():
|
||||
instance = instance_generator(groups=groups, hostname=hostname)
|
||||
worker_queues = worker_queues_generator(_worker_queues)
|
||||
with mock.patch('awx.main.utils.ha.settings.AWX_CELERY_QUEUES_STATIC', static_queues):
|
||||
(added_queues, removed_queues) = _add_remove_celery_worker_queues(mock_app, instance, worker_queues, hostname)
|
||||
(added_queues, removed_queues) = _add_remove_celery_worker_queues(mock_app, [instance], worker_queues, hostname)
|
||||
assert set(added_queues) == set(added_expected)
|
||||
assert set(removed_queues) == set(removed_expected)
|
||||
|
||||
|
||||
@ -10,26 +10,27 @@ from django.conf import settings
|
||||
from awx.main.models import Instance
|
||||
|
||||
|
||||
def _add_remove_celery_worker_queues(app, instance, worker_queues, worker_name):
|
||||
def _add_remove_celery_worker_queues(app, controlled_instances, worker_queues, worker_name):
|
||||
removed_queues = []
|
||||
added_queues = []
|
||||
ig_names = set(instance.rampart_groups.values_list('name', flat=True))
|
||||
ig_names.add("tower_instance_router")
|
||||
ig_names = set(['tower_instance_router'])
|
||||
hostnames = set([instance.hostname for instance in controlled_instances])
|
||||
for instance in controlled_instances:
|
||||
ig_names.update(instance.rampart_groups.values_list('name', flat=True))
|
||||
worker_queue_names = set([q['name'] for q in worker_queues])
|
||||
|
||||
|
||||
# Remove queues that aren't in the instance group
|
||||
for queue in worker_queues:
|
||||
if queue['name'] in settings.AWX_CELERY_QUEUES_STATIC or \
|
||||
queue['alias'] in settings.AWX_CELERY_QUEUES_STATIC:
|
||||
continue
|
||||
|
||||
if queue['name'] not in ig_names | set([instance.hostname]) or not instance.enabled:
|
||||
if queue['name'] not in ig_names | hostnames or not instance.enabled:
|
||||
app.control.cancel_consumer(queue['name'], reply=True, destination=[worker_name])
|
||||
removed_queues.append(queue['name'])
|
||||
|
||||
# Add queues for instance and instance groups
|
||||
for queue_name in ig_names | set([instance.hostname]):
|
||||
for queue_name in ig_names | hostnames:
|
||||
if queue_name not in worker_queue_names:
|
||||
app.control.add_consumer(queue_name, reply=True, destination=[worker_name])
|
||||
added_queues.append(queue_name)
|
||||
@ -59,13 +60,19 @@ def update_celery_worker_routes(instance, conf):
|
||||
|
||||
def register_celery_worker_queues(app, celery_worker_name):
|
||||
instance = Instance.objects.me()
|
||||
controlled_instances = [instance]
|
||||
if instance.is_controller():
|
||||
controlled_instances.extend(Instance.objects.filter(
|
||||
rampart_groups__controller__instances__hostname=instance.hostname
|
||||
))
|
||||
added_queues = []
|
||||
removed_queues = []
|
||||
|
||||
celery_host_queues = app.control.inspect([celery_worker_name]).active_queues()
|
||||
|
||||
celery_worker_queues = celery_host_queues[celery_worker_name] if celery_host_queues else []
|
||||
(added_queues, removed_queues) = _add_remove_celery_worker_queues(app, instance, celery_worker_queues, celery_worker_name)
|
||||
(added_queues, removed_queues) = _add_remove_celery_worker_queues(app, controlled_instances,
|
||||
celery_worker_queues, celery_worker_name)
|
||||
|
||||
return (instance, removed_queues, added_queues)
|
||||
return (controlled_instances, removed_queues, added_queues)
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ requirements/requirements_dev_uninstall.txt \
|
||||
RUN yum -y update && yum -y install curl epel-release && yum -y install https://centos7.iuscommunity.org/ius-release.rpm
|
||||
RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
|
||||
RUN yum -y localinstall http://download.postgresql.org/pub/repos/yum/9.4/redhat/rhel-6-x86_64/pgdg-centos94-9.4-3.noarch.rpm
|
||||
RUN yum -y update && yum -y install openssh-server ansible mg vim tmux git2u-core mercurial subversion python-devel python-psycopg2 make postgresql postgresql-devel nginx nodejs python-psutil libxml2-devel libxslt-devel libstdc++.so.6 gcc cyrus-sasl-devel cyrus-sasl openldap-devel libffi-devel zeromq-devel python-pip xmlsec1-devel swig krb5-devel xmlsec1-openssl xmlsec1 xmlsec1-openssl-devel libtool-ltdl-devel rabbitmq-server bubblewrap zanata-python-client gettext gcc-c++ libcurl-devel python-pycurl bzip2 python-crypto
|
||||
RUN yum -y update && yum -y install openssh-server ansible mg vim tmux git2u-core mercurial subversion python-devel python-psycopg2 make postgresql postgresql-devel nginx nodejs python-psutil libxml2-devel libxslt-devel libstdc++.so.6 gcc cyrus-sasl-devel cyrus-sasl openldap-devel libffi-devel zeromq-devel python-pip xmlsec1-devel swig krb5-devel xmlsec1-openssl xmlsec1 xmlsec1-openssl-devel libtool-ltdl-devel rabbitmq-server bubblewrap zanata-python-client gettext gcc-c++ libcurl-devel python-pycurl bzip2 python-crypto rsync
|
||||
RUN pip install virtualenv
|
||||
RUN /usr/bin/ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa
|
||||
RUN mkdir -p /data/db
|
||||
|
||||
@ -5,8 +5,6 @@ ADD Makefile /tmp/Makefile
|
||||
RUN mkdir /tmp/requirements
|
||||
ADD requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt requirements/requirements_ansible_uninstall.txt requirements/requirements_isolated.txt /tmp/requirements/
|
||||
RUN yum -y update && yum -y install curl epel-release
|
||||
RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
|
||||
RUN yum -y localinstall http://download.postgresql.org/pub/repos/yum/9.4/redhat/rhel-6-x86_64/pgdg-centos94-9.4-3.noarch.rpm
|
||||
RUN yum -y update && yum -y install openssh-server ansible mg vim tmux git python-devel python-psycopg2 make python-psutil libxml2-devel libxslt-devel libstdc++.so.6 gcc cyrus-sasl-devel cyrus-sasl openldap-devel libffi-devel zeromq-devel python-pip xmlsec1-devel swig krb5-devel xmlsec1-openssl xmlsec1 xmlsec1-openssl-devel libtool-ltdl-devel bubblewrap zanata-python-client gettext gcc-c++ libcurl-devel python-pycurl bzip2
|
||||
RUN pip install virtualenv
|
||||
WORKDIR /tmp
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user