External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-05-17 06:17:36 -02:30
Code Issues Packages Projects Releases Wiki Activity

Rollback celery

Browse Source
This commit is contained in:
Wayne Witzel III
2018-02-22 09:32:20 -05:00
parent 90bb43ce74
commit 91c6d406c5
10 changed files with 40 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
awx/__init__.py
View File

@@ -7,7 +7,7 @@ import sys
import warnings import warnings
from pkg_resources import get_distribution from pkg_resources import get_distribution
from .celery import app as celery_app from .celery import app as celery_app # noqa
__version__ = get_distribution('awx').version __version__ = get_distribution('awx').version

6
awx/celery.py
View File

@@ -1,3 +1,4 @@
# Copyright (c) 2017 Ansible, Inc. # Copyright (c) 2017 Ansible, Inc.
# All Rights Reserved. # All Rights Reserved.
@@ -5,6 +6,7 @@ from __future__ import absolute_import, unicode_literals
import os import os
from celery import Celery from celery import Celery
from django.conf import settings # noqa
try: try:
@@ -16,8 +18,8 @@ except ImportError: # pragma: no cover
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE) os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
app = Celery('awx') app = Celery('awx')
app.config_from_object('django.conf:settings', namespace='CELERY') app.config_from_object('django.conf:settings')
app.autodiscover_tasks() app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
if __name__ == '__main__': if __name__ == '__main__':
app.start() app.start()

2
awx/main/management/commands/run_callback_receiver.py
View File

@@ -208,7 +208,7 @@ class Command(BaseCommand):
help = 'Launch the job callback receiver' help = 'Launch the job callback receiver'
def handle(self, *arg, **options): def handle(self, *arg, **options):
with Connection(settings.CELERY_BROKER_URL) as conn: with Connection(settings.BROKER_URL) as conn:
try: try:
worker = CallbackBrokerWorker(conn) worker = CallbackBrokerWorker(conn)
worker.run() worker.run()

8
awx/main/models/unified_jobs.py
View File

@@ -28,7 +28,7 @@ from rest_framework.exceptions import ParseError
from polymorphic.models import PolymorphicModel from polymorphic.models import PolymorphicModel
# Django-Celery # Django-Celery
from django_celery_results.models import TaskResult from djcelery.models import TaskMeta
# AWX # AWX
from awx.main.models.base import * # noqa from awx.main.models.base import * # noqa
@@ -1093,8 +1093,8 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
def celery_task(self): def celery_task(self):
try: try:
if self.celery_task_id: if self.celery_task_id:
return TaskResult.objects.get(task_id=self.celery_task_id) return TaskMeta.objects.get(task_id=self.celery_task_id)
except TaskResult.DoesNotExist: except TaskMeta.DoesNotExist:
pass pass
def get_passwords_needed_to_start(self): def get_passwords_needed_to_start(self):
@@ -1335,7 +1335,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
cancel_fields.append('job_explanation') cancel_fields.append('job_explanation')
self.save(update_fields=cancel_fields) self.save(update_fields=cancel_fields)
self.websocket_emit_status("canceled") self.websocket_emit_status("canceled")
if settings.CELERY_BROKER_URL.startswith('amqp://'): if settings.BROKER_URL.startswith('amqp://'):
self._force_cancel() self._force_cancel()
return self.cancel_flag return self.cancel_flag

2
awx/main/queue.py
View File

@@ -19,7 +19,7 @@ __all__ = ['CallbackQueueDispatcher']
class CallbackQueueDispatcher(object): class CallbackQueueDispatcher(object):
def __init__(self): def __init__(self):
self.callback_connection = getattr(settings, 'CELERY_BROKER_URL', None) self.callback_connection = getattr(settings, 'BROKER_URL', None)
self.connection_queue = getattr(settings, 'CALLBACK_QUEUE', '') self.connection_queue = getattr(settings, 'CALLBACK_QUEUE', '')
self.connection = None self.connection = None
self.exchange = None self.exchange = None

2
awx/main/scheduler/task_manager.py
View File

@@ -133,7 +133,7 @@ class TaskManager():
def get_active_tasks(self): def get_active_tasks(self):
if not hasattr(settings, 'IGNORE_CELERY_INSPECTOR'): if not hasattr(settings, 'IGNORE_CELERY_INSPECTOR'):
app = Celery('awx') app = Celery('awx')
app.config_from_object('django.conf:settings', namespace='CELERY') app.config_from_object('django.conf:settings')
inspector = Inspect(app=app) inspector = Inspect(app=app)
active_task_queues = inspector.active() active_task_queues = inspector.active()
else: else:

10
awx/main/tasks.py
View File

@@ -48,7 +48,6 @@ import six
# AWX # AWX
from awx import __version__ as awx_application_version from awx import __version__ as awx_application_version
from awx import celery_app
from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS
from awx.main.models import * # noqa from awx.main.models import * # noqa
from awx.main.models.unified_jobs import ACTIVE_STATES from awx.main.models.unified_jobs import ACTIVE_STATES
@@ -207,7 +206,7 @@ def handle_ha_toplogy_changes(self):
instance = Instance.objects.me() instance = Instance.objects.me()
logger.debug("Reconfigure celeryd queues task on host {}".format(self.request.hostname)) logger.debug("Reconfigure celeryd queues task on host {}".format(self.request.hostname))
awx_app = Celery('awx') awx_app = Celery('awx')
awx_app.config_from_object('django.conf:settings', namespace='CELERY') awx_app.config_from_object('django.conf:settings')
instances, removed_queues, added_queues = register_celery_worker_queues(awx_app, self.request.hostname) instances, removed_queues, added_queues = register_celery_worker_queues(awx_app, self.request.hostname)
for instance in instances: for instance in instances:
logger.info("Workers on tower node '{}' removed from queues {} and added to queues {}" logger.info("Workers on tower node '{}' removed from queues {} and added to queues {}"
@@ -2332,10 +2331,3 @@ def deep_copy_model_obj(
importlib.import_module(permission_check_func[0]), permission_check_func[1] importlib.import_module(permission_check_func[0]), permission_check_func[1]
), permission_check_func[2]) ), permission_check_func[2])
permission_check_func(creater, copy_mapping.values()) permission_check_func(creater, copy_mapping.values())
celery_app.register_task(RunJob())
celery_app.register_task(RunProjectUpdate())
celery_app.register_task(RunInventoryUpdate())
celery_app.register_task(RunAdHocCommand())
celery_app.register_task(RunSystemJob())

27
awx/settings/defaults.py
View File

@@ -5,6 +5,7 @@ import os
import re # noqa import re # noqa
import sys import sys
import ldap import ldap
import djcelery
from datetime import timedelta from datetime import timedelta
from kombu import Queue, Exchange from kombu import Queue, Exchange
@@ -255,7 +256,7 @@ INSTALLED_APPS = (
'django.contrib.staticfiles', 'django.contrib.staticfiles',
'rest_framework', 'rest_framework',
'django_extensions', 'django_extensions',
'django_celery_results', 'djcelery',
'channels', 'channels',
'polymorphic', 'polymorphic',
'taggit', 'taggit',
@@ -433,28 +434,30 @@ DEVSERVER_DEFAULT_PORT = '8013'
# Set default ports for live server tests. # Set default ports for live server tests.
os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199') os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199')
djcelery.setup_loader()
BROKER_POOL_LIMIT = None BROKER_POOL_LIMIT = None
CELERY_BROKER_URL = 'amqp://guest:guest@localhost:5672//' BROKER_URL = 'amqp://guest:guest@localhost:5672//'
CELERY_EVENT_QUEUE_TTL = 5 CELERY_EVENT_QUEUE_TTL = 5
CELERY_TASK_DEFAULT_QUEUE = 'tower' CELERY_DEFAULT_QUEUE = 'tower'
CELERY_TASK_SERIALIZER = 'json' CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json'] CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_TRACK_STARTED = True CELERY_TRACK_STARTED = True
CELERY_TASK_TIME_LIMIT = None CELERYD_TASK_TIME_LIMIT = None
CELERY_TASK_SOFT_TIME_LIMIT = None CELERYD_TASK_SOFT_TIME_LIMIT = None
CELERY_WORKER_POOL_RESTARTS = True CELERYD_POOL_RESTARTS = True
CELERY_BEAT_SCHEDULER = 'celery.beat.PersistentScheduler' CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'
CELERY_BEAT_MAX_LOOP_INTERVAL = 60
CELERY_RESULT_BACKEND = 'django-db'
CELERY_IMPORTS = ('awx.main.scheduler.tasks',) CELERY_IMPORTS = ('awx.main.scheduler.tasks',)
CELERY_TASK_QUEUES = ( CELERY_QUEUES = (
Queue('tower', Exchange('tower'), routing_key='tower'), Queue('tower', Exchange('tower'), routing_key='tower'),
Broadcast('tower_broadcast_all') Broadcast('tower_broadcast_all')
) )
CELERY_TASK_ROUTES = {} CELERY_TASK_ROUTES = {}
CELERY_BEAT_SCHEDULE = { CELERYBEAT_SCHEDULER = 'celery.beat.PersistentScheduler'
CELERYBEAT_MAX_LOOP_INTERVAL = 60
CELERYBEAT_SCHEDULE = {
'tower_scheduler': { 'tower_scheduler': {
'task': 'awx.main.tasks.awx_periodic_scheduler', 'task': 'awx.main.tasks.awx_periodic_scheduler',
'schedule': timedelta(seconds=30), 'schedule': timedelta(seconds=30),

5
requirements/requirements.in
View File

@@ -7,12 +7,11 @@ backports.ssl-match-hostname==3.5.0.1
boto==2.46.1 boto==2.46.1
boto3==1.4.4 boto3==1.4.4
channels==1.1.8 channels==1.1.8
celery==4.1 celery==3.1.25
daphne==1.3.0 daphne==1.3.0
Django==1.11.7 Django==1.11.7
django-auth-ldap==1.2.8 django-auth-ldap==1.2.8
django-celery-beat==1.1.0 django-celery==3.2.2
django-celery-results==1.0.1
django-crum==0.7.1 django-crum==0.7.1
django-extensions==1.7.8 django-extensions==1.7.8
django-jsonfield==1.0.1 django-jsonfield==1.0.1

22
requirements/requirements.txt
View File

@@ -5,7 +5,8 @@
# pip-compile --output-file requirements/requirements.txt requirements/requirements.in # pip-compile --output-file requirements/requirements.txt requirements/requirements.in
# #
adal==0.4.5 # via msrestazure adal==0.4.5 # via msrestazure
amqp==2.2.2 # via kombu amqp==1.4.9 # via kombu
anyjson==0.3.3 # via kombu
apache-libcloud==2.0.0 apache-libcloud==2.0.0
appdirs==1.4.2 appdirs==1.4.2
asgi-amqp==1.0.3 asgi-amqp==1.0.3
@@ -36,12 +37,12 @@ babel==2.3.4 # via osc-lib, oslo.i18n, python-cinderclient, python-
backports.functools-lru-cache==1.4 # via jaraco.functools backports.functools-lru-cache==1.4 # via jaraco.functools
backports.ssl-match-hostname==3.5.0.1 backports.ssl-match-hostname==3.5.0.1
baron==0.6.6 # via redbaron baron==0.6.6 # via redbaron
billiard==3.5.0.3 # via celery billiard==3.3.0.23 # via celery
boto3==1.4.4 boto3==1.4.4
boto==2.46.1 boto==2.46.1
botocore==1.5.72 # via boto3, s3transfer botocore==1.5.72 # via boto3, s3transfer
celery==4.1 celery==3.1.25
#certifi==2017.11.5 # via msrest #certifi==2018.1.18 # via msrest
cffi==1.10.0 # via cryptography cffi==1.10.0 # via cryptography
channels==1.1.8 channels==1.1.8
cliff==2.7.0 # via osc-lib, python-designateclient, python-neutronclient, python-openstackclient cliff==2.7.0 # via osc-lib, python-designateclient, python-neutronclient, python-openstackclient
@@ -54,8 +55,7 @@ decorator==4.0.11 # via shade
defusedxml==0.4.1 # via python-saml defusedxml==0.4.1 # via python-saml
deprecation==1.0.1 # via openstacksdk deprecation==1.0.1 # via openstacksdk
django-auth-ldap==1.2.8 django-auth-ldap==1.2.8
django-celery-beat==1.1.0 django-celery==3.2.2
django-celery-results==1.0.1
django-crum==0.7.1 django-crum==0.7.1
django-extensions==1.7.8 django-extensions==1.7.8
django-jsonfield==1.0.1 django-jsonfield==1.0.1
@@ -72,7 +72,6 @@ djangorestframework==3.7.3
#docutils==0.14 # via botocore #docutils==0.14 # via botocore
dogpile.cache==0.6.3 # via python-ironicclient, shade dogpile.cache==0.6.3 # via python-ironicclient, shade
enum34==1.1.6 # via cryptography, msrest enum34==1.1.6 # via cryptography, msrest
ephem==3.7.6.0 # via django-celery-beat
funcsigs==1.0.2 # via debtcollector, oslo.utils funcsigs==1.0.2 # via debtcollector, oslo.utils
functools32==3.2.3.post2 # via jsonschema functools32==3.2.3.post2 # via jsonschema
futures==3.1.1 # via azure-storage, requests-futures, s3transfer, shade futures==3.1.1 # via azure-storage, requests-futures, s3transfer, shade
@@ -96,18 +95,18 @@ jaraco.stream==1.1.2 # via irc
jaraco.text==1.9.2 # via irc, jaraco.collections jaraco.text==1.9.2 # via irc, jaraco.collections
jmespath==0.9.3 # via boto3, botocore, shade jmespath==0.9.3 # via boto3, botocore, shade
jsonpatch==1.16 # via openstacksdk, shade, warlock jsonpatch==1.16 # via openstacksdk, shade, warlock
jsonpickle==0.9.5 # via asgi_amqp jsonpickle==0.9.5 # via asgi-amqp
jsonpointer==1.10 # via jsonpatch jsonpointer==1.10 # via jsonpatch
jsonschema==2.6.0 jsonschema==2.6.0
keyring==10.3.3 # via msrestazure keyring==10.3.3 # via msrestazure
keystoneauth1==2.21.0 # via openstacksdk, os-client-config, osc-lib, python-cinderclient, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient, shade keystoneauth1==2.21.0 # via openstacksdk, os-client-config, osc-lib, python-cinderclient, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient, shade
kombu==4.1.0 # via celery kombu==3.0.37 # via asgi-amqp, celery
lxml==3.8.0 # via dm.xmlsec.binding, pyvmomi lxml==3.8.0 # via dm.xmlsec.binding, pyvmomi
m2crypto==0.25.1 m2crypto==0.25.1
markdown==2.6.7 markdown==2.6.7
monotonic==1.3 # via oslo.utils monotonic==1.3 # via oslo.utils
more-itertools==3.2.0 # via irc, jaraco.functools, jaraco.itertools more-itertools==3.2.0 # via irc, jaraco.functools, jaraco.itertools
msgpack-python==0.4.8 # via oslo.serialization msgpack-python==0.4.8 # via asgi-amqp, oslo.serialization
msrest==0.4.10 # via azure-common, msrestazure msrest==0.4.10 # via azure-common, msrestazure
msrestazure==0.4.9 # via azure-common msrestazure==0.4.9 # via azure-common
munch==2.1.1 # via shade munch==2.1.1 # via shade
@@ -169,7 +168,7 @@ secretstorage==2.3.1 # via keyring
service-identity==16.0.0 service-identity==16.0.0
shade==1.20.0 shade==1.20.0
simplejson==3.11.1 # via osc-lib, python-cinderclient, python-neutronclient, python-novaclient simplejson==3.11.1 # via osc-lib, python-cinderclient, python-neutronclient, python-novaclient
six==1.10.0 # via asgiref, autobahn, automat, cliff, cmd2, cryptography, debtcollector, django-extensions, irc, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, keystoneauth1, more-itertools, munch, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, pygerduty, pyopenssl, pyrad, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, pyvmomi, shade, slackclient, social-auth-app-django, social-auth-core, stevedore, tacacs-plus, tempora, twilio, txaio, warlock, websocket-client six==1.10.0 # via asgi-amqp, asgiref, autobahn, automat, cliff, cmd2, cryptography, debtcollector, django-extensions, irc, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, keystoneauth1, more-itertools, munch, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, pygerduty, pyopenssl, pyrad, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, pyvmomi, shade, slackclient, social-auth-app-django, social-auth-core, stevedore, tacacs-plus, tempora, twilio, txaio, warlock, websocket-client
slackclient==1.0.6 slackclient==1.0.6
social-auth-app-django==2.0.0 social-auth-app-django==2.0.0
social-auth-core==1.5.0 social-auth-core==1.5.0
@@ -183,7 +182,6 @@ txaio==2.8.2 # via autobahn
typing==3.6.2 # via m2crypto typing==3.6.2 # via m2crypto
unicodecsv==0.14.1 # via cliff unicodecsv==0.14.1 # via cliff
uwsgi==2.0.14 uwsgi==2.0.14
vine==1.1.4 # via amqp
warlock==1.2.0 # via python-glanceclient warlock==1.2.0 # via python-glanceclient
websocket-client==0.44.0 # via slackclient websocket-client==0.44.0 # via slackclient
wrapt==1.10.10 # via debtcollector, positional, python-glanceclient wrapt==1.10.10 # via debtcollector, positional, python-glanceclient