From 91c6d406c58919007eabe5c1c975730417c0d9bf Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Thu, 22 Feb 2018 09:32:20 -0500 Subject: [PATCH] Rollback celery --- awx/__init__.py | 2 +- awx/celery.py | 6 +++-- .../commands/run_callback_receiver.py | 2 +- awx/main/models/unified_jobs.py | 8 +++--- awx/main/queue.py | 2 +- awx/main/scheduler/task_manager.py | 2 +- awx/main/tasks.py | 10 +------ awx/settings/defaults.py | 27 ++++++++++--------- requirements/requirements.in | 5 ++-- requirements/requirements.txt | 22 +++++++-------- 10 files changed, 40 insertions(+), 46 deletions(-) diff --git a/awx/__init__.py b/awx/__init__.py index aa38587de3..f107340385 100644 --- a/awx/__init__.py +++ b/awx/__init__.py @@ -7,7 +7,7 @@ import sys import warnings from pkg_resources import get_distribution -from .celery import app as celery_app +from .celery import app as celery_app # noqa __version__ = get_distribution('awx').version diff --git a/awx/celery.py b/awx/celery.py index d1854d9487..124a2a4fd4 100644 --- a/awx/celery.py +++ b/awx/celery.py @@ -1,3 +1,4 @@ + # Copyright (c) 2017 Ansible, Inc. # All Rights Reserved. @@ -5,6 +6,7 @@ from __future__ import absolute_import, unicode_literals import os from celery import Celery +from django.conf import settings # noqa try: @@ -16,8 +18,8 @@ except ImportError: # pragma: no cover os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE) app = Celery('awx') -app.config_from_object('django.conf:settings', namespace='CELERY') -app.autodiscover_tasks() +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) if __name__ == '__main__': app.start() diff --git a/awx/main/management/commands/run_callback_receiver.py b/awx/main/management/commands/run_callback_receiver.py index adf558631e..fbbcb4f917 100644 --- a/awx/main/management/commands/run_callback_receiver.py +++ b/awx/main/management/commands/run_callback_receiver.py @@ -208,7 +208,7 @@ class Command(BaseCommand): help = 'Launch the job callback receiver' def handle(self, *arg, **options): - with Connection(settings.CELERY_BROKER_URL) as conn: + with Connection(settings.BROKER_URL) as conn: try: worker = CallbackBrokerWorker(conn) worker.run() diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 16c043a991..24e205d371 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -28,7 +28,7 @@ from rest_framework.exceptions import ParseError from polymorphic.models import PolymorphicModel # Django-Celery -from django_celery_results.models import TaskResult +from djcelery.models import TaskMeta # AWX from awx.main.models.base import * # noqa @@ -1093,8 +1093,8 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique def celery_task(self): try: if self.celery_task_id: - return TaskResult.objects.get(task_id=self.celery_task_id) - except TaskResult.DoesNotExist: + return TaskMeta.objects.get(task_id=self.celery_task_id) + except TaskMeta.DoesNotExist: pass def get_passwords_needed_to_start(self): @@ -1335,7 +1335,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique cancel_fields.append('job_explanation') self.save(update_fields=cancel_fields) self.websocket_emit_status("canceled") - if settings.CELERY_BROKER_URL.startswith('amqp://'): + if settings.BROKER_URL.startswith('amqp://'): self._force_cancel() return self.cancel_flag diff --git a/awx/main/queue.py b/awx/main/queue.py index 3f68c68c2a..8f5b680f84 100644 --- a/awx/main/queue.py +++ b/awx/main/queue.py @@ -19,7 +19,7 @@ __all__ = ['CallbackQueueDispatcher'] class CallbackQueueDispatcher(object): def __init__(self): - self.callback_connection = getattr(settings, 'CELERY_BROKER_URL', None) + self.callback_connection = getattr(settings, 'BROKER_URL', None) self.connection_queue = getattr(settings, 'CALLBACK_QUEUE', '') self.connection = None self.exchange = None diff --git a/awx/main/scheduler/task_manager.py b/awx/main/scheduler/task_manager.py index f0c686cd1f..ea46146b60 100644 --- a/awx/main/scheduler/task_manager.py +++ b/awx/main/scheduler/task_manager.py @@ -133,7 +133,7 @@ class TaskManager(): def get_active_tasks(self): if not hasattr(settings, 'IGNORE_CELERY_INSPECTOR'): app = Celery('awx') - app.config_from_object('django.conf:settings', namespace='CELERY') + app.config_from_object('django.conf:settings') inspector = Inspect(app=app) active_task_queues = inspector.active() else: diff --git a/awx/main/tasks.py b/awx/main/tasks.py index b15d0b73bd..b0c1483de3 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -48,7 +48,6 @@ import six # AWX from awx import __version__ as awx_application_version -from awx import celery_app from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS from awx.main.models import * # noqa from awx.main.models.unified_jobs import ACTIVE_STATES @@ -207,7 +206,7 @@ def handle_ha_toplogy_changes(self): instance = Instance.objects.me() logger.debug("Reconfigure celeryd queues task on host {}".format(self.request.hostname)) awx_app = Celery('awx') - awx_app.config_from_object('django.conf:settings', namespace='CELERY') + awx_app.config_from_object('django.conf:settings') instances, removed_queues, added_queues = register_celery_worker_queues(awx_app, self.request.hostname) for instance in instances: logger.info("Workers on tower node '{}' removed from queues {} and added to queues {}" @@ -2332,10 +2331,3 @@ def deep_copy_model_obj( importlib.import_module(permission_check_func[0]), permission_check_func[1] ), permission_check_func[2]) permission_check_func(creater, copy_mapping.values()) - - -celery_app.register_task(RunJob()) -celery_app.register_task(RunProjectUpdate()) -celery_app.register_task(RunInventoryUpdate()) -celery_app.register_task(RunAdHocCommand()) -celery_app.register_task(RunSystemJob()) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index a0618c238d..8268849c61 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -5,6 +5,7 @@ import os import re # noqa import sys import ldap +import djcelery from datetime import timedelta from kombu import Queue, Exchange @@ -255,7 +256,7 @@ INSTALLED_APPS = ( 'django.contrib.staticfiles', 'rest_framework', 'django_extensions', - 'django_celery_results', + 'djcelery', 'channels', 'polymorphic', 'taggit', @@ -433,28 +434,30 @@ DEVSERVER_DEFAULT_PORT = '8013' # Set default ports for live server tests. os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199') +djcelery.setup_loader() + BROKER_POOL_LIMIT = None -CELERY_BROKER_URL = 'amqp://guest:guest@localhost:5672//' +BROKER_URL = 'amqp://guest:guest@localhost:5672//' CELERY_EVENT_QUEUE_TTL = 5 -CELERY_TASK_DEFAULT_QUEUE = 'tower' +CELERY_DEFAULT_QUEUE = 'tower' CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TASK_TRACK_STARTED = True -CELERY_TASK_TIME_LIMIT = None -CELERY_TASK_SOFT_TIME_LIMIT = None -CELERY_WORKER_POOL_RESTARTS = True -CELERY_BEAT_SCHEDULER = 'celery.beat.PersistentScheduler' -CELERY_BEAT_MAX_LOOP_INTERVAL = 60 -CELERY_RESULT_BACKEND = 'django-db' +CELERY_TRACK_STARTED = True +CELERYD_TASK_TIME_LIMIT = None +CELERYD_TASK_SOFT_TIME_LIMIT = None +CELERYD_POOL_RESTARTS = True +CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend' CELERY_IMPORTS = ('awx.main.scheduler.tasks',) -CELERY_TASK_QUEUES = ( +CELERY_QUEUES = ( Queue('tower', Exchange('tower'), routing_key='tower'), Broadcast('tower_broadcast_all') ) CELERY_TASK_ROUTES = {} -CELERY_BEAT_SCHEDULE = { +CELERYBEAT_SCHEDULER = 'celery.beat.PersistentScheduler' +CELERYBEAT_MAX_LOOP_INTERVAL = 60 +CELERYBEAT_SCHEDULE = { 'tower_scheduler': { 'task': 'awx.main.tasks.awx_periodic_scheduler', 'schedule': timedelta(seconds=30), diff --git a/requirements/requirements.in b/requirements/requirements.in index a8caad4f90..14ea9f93e6 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -7,12 +7,11 @@ backports.ssl-match-hostname==3.5.0.1 boto==2.46.1 boto3==1.4.4 channels==1.1.8 -celery==4.1 +celery==3.1.25 daphne==1.3.0 Django==1.11.7 django-auth-ldap==1.2.8 -django-celery-beat==1.1.0 -django-celery-results==1.0.1 +django-celery==3.2.2 django-crum==0.7.1 django-extensions==1.7.8 django-jsonfield==1.0.1 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 73d2ae0b1e..a491681908 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -5,7 +5,8 @@ # pip-compile --output-file requirements/requirements.txt requirements/requirements.in # adal==0.4.5 # via msrestazure -amqp==2.2.2 # via kombu +amqp==1.4.9 # via kombu +anyjson==0.3.3 # via kombu apache-libcloud==2.0.0 appdirs==1.4.2 asgi-amqp==1.0.3 @@ -36,12 +37,12 @@ babel==2.3.4 # via osc-lib, oslo.i18n, python-cinderclient, python- backports.functools-lru-cache==1.4 # via jaraco.functools backports.ssl-match-hostname==3.5.0.1 baron==0.6.6 # via redbaron -billiard==3.5.0.3 # via celery +billiard==3.3.0.23 # via celery boto3==1.4.4 boto==2.46.1 botocore==1.5.72 # via boto3, s3transfer -celery==4.1 -#certifi==2017.11.5 # via msrest +celery==3.1.25 +#certifi==2018.1.18 # via msrest cffi==1.10.0 # via cryptography channels==1.1.8 cliff==2.7.0 # via osc-lib, python-designateclient, python-neutronclient, python-openstackclient @@ -54,8 +55,7 @@ decorator==4.0.11 # via shade defusedxml==0.4.1 # via python-saml deprecation==1.0.1 # via openstacksdk django-auth-ldap==1.2.8 -django-celery-beat==1.1.0 -django-celery-results==1.0.1 +django-celery==3.2.2 django-crum==0.7.1 django-extensions==1.7.8 django-jsonfield==1.0.1 @@ -72,7 +72,6 @@ djangorestframework==3.7.3 #docutils==0.14 # via botocore dogpile.cache==0.6.3 # via python-ironicclient, shade enum34==1.1.6 # via cryptography, msrest -ephem==3.7.6.0 # via django-celery-beat funcsigs==1.0.2 # via debtcollector, oslo.utils functools32==3.2.3.post2 # via jsonschema futures==3.1.1 # via azure-storage, requests-futures, s3transfer, shade @@ -96,18 +95,18 @@ jaraco.stream==1.1.2 # via irc jaraco.text==1.9.2 # via irc, jaraco.collections jmespath==0.9.3 # via boto3, botocore, shade jsonpatch==1.16 # via openstacksdk, shade, warlock -jsonpickle==0.9.5 # via asgi_amqp +jsonpickle==0.9.5 # via asgi-amqp jsonpointer==1.10 # via jsonpatch jsonschema==2.6.0 keyring==10.3.3 # via msrestazure keystoneauth1==2.21.0 # via openstacksdk, os-client-config, osc-lib, python-cinderclient, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient, shade -kombu==4.1.0 # via celery +kombu==3.0.37 # via asgi-amqp, celery lxml==3.8.0 # via dm.xmlsec.binding, pyvmomi m2crypto==0.25.1 markdown==2.6.7 monotonic==1.3 # via oslo.utils more-itertools==3.2.0 # via irc, jaraco.functools, jaraco.itertools -msgpack-python==0.4.8 # via oslo.serialization +msgpack-python==0.4.8 # via asgi-amqp, oslo.serialization msrest==0.4.10 # via azure-common, msrestazure msrestazure==0.4.9 # via azure-common munch==2.1.1 # via shade @@ -169,7 +168,7 @@ secretstorage==2.3.1 # via keyring service-identity==16.0.0 shade==1.20.0 simplejson==3.11.1 # via osc-lib, python-cinderclient, python-neutronclient, python-novaclient -six==1.10.0 # via asgiref, autobahn, automat, cliff, cmd2, cryptography, debtcollector, django-extensions, irc, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, keystoneauth1, more-itertools, munch, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, pygerduty, pyopenssl, pyrad, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, pyvmomi, shade, slackclient, social-auth-app-django, social-auth-core, stevedore, tacacs-plus, tempora, twilio, txaio, warlock, websocket-client +six==1.10.0 # via asgi-amqp, asgiref, autobahn, automat, cliff, cmd2, cryptography, debtcollector, django-extensions, irc, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, keystoneauth1, more-itertools, munch, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, pygerduty, pyopenssl, pyrad, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, pyvmomi, shade, slackclient, social-auth-app-django, social-auth-core, stevedore, tacacs-plus, tempora, twilio, txaio, warlock, websocket-client slackclient==1.0.6 social-auth-app-django==2.0.0 social-auth-core==1.5.0 @@ -183,7 +182,6 @@ txaio==2.8.2 # via autobahn typing==3.6.2 # via m2crypto unicodecsv==0.14.1 # via cliff uwsgi==2.0.14 -vine==1.1.4 # via amqp warlock==1.2.0 # via python-glanceclient websocket-client==0.44.0 # via slackclient wrapt==1.10.10 # via debtcollector, positional, python-glanceclient