External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-03 09:48:51 -03:30
Code Issues Packages Projects Releases Wiki Activity

replace celery task decorators with a kombu-based publisher

Browse Source 
this commit implements the bulk of `awx-manage run_dispatcher`, a new
command that binds to RabbitMQ via kombu and balances messages across
a pool of workers that are similar to celeryd workers in spirit.
Specifically, this includes:

- a new decorator, `awx.main.dispatch.task`, which can be used to
  decorate functions or classes so that they can be designated as
  "Tasks"
- support for fanout/broadcast tasks (at this point in time, only
  `conf.Setting` memcached flushes use this functionality)
- support for job reaping
- support for success/failure hooks for job runs (i.e.,
  `handle_work_success` and `handle_work_error`)
- support for auto scaling worker pool that scale processes up and down
  on demand
- minimal support for RPC, such as status checks and pool recycle/reload
This commit is contained in:
Ryan Petrello
2018-08-08 13:41:07 -04:00
parent da74f1d01f
commit ff1e8cc356
54 changed files with 1606 additions and 1147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
awx/settings/defaults.py
View File

@@ -4,7 +4,6 @@
import os
import re # noqa
import sys
import djcelery
import six
from datetime import timedelta
@@ -26,6 +25,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(__file__))
def is_testing(argv=None):
import sys
'''Return True if running django or py.test unit tests.'''
if 'PYTEST_CURRENT_TEST' in os.environ.keys():
return True
argv = sys.argv if argv is None else argv
if len(argv) >= 1 and ('py.test' in argv[0] or 'py/test.py' in argv[0]):
return True
@@ -60,7 +61,7 @@ DATABASES = {
'NAME': os.path.join(BASE_DIR, 'awx.sqlite3'),
'ATOMIC_REQUESTS': True,
'TEST': {
# Test database cannot be :memory: for celery/inventory tests.
# Test database cannot be :memory: for inventory tests.
'NAME': os.path.join(BASE_DIR, 'awx_test.sqlite3'),
},
}
@@ -280,7 +281,6 @@ INSTALLED_APPS = (
'oauth2_provider',
'rest_framework',
'django_extensions',
'djcelery',
'channels',
'polymorphic',
'taggit',
@@ -459,40 +459,9 @@ DEVSERVER_DEFAULT_PORT = '8013'
# Set default ports for live server tests.
os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199')
djcelery.setup_loader()
BROKER_POOL_LIMIT = None
BROKER_URL = 'amqp://guest:guest@localhost:5672//'
CELERY_EVENT_QUEUE_TTL = 5
CELERY_DEFAULT_QUEUE = 'awx_private_queue'
CELERY_DEFAULT_EXCHANGE = 'awx_private_queue'
CELERY_DEFAULT_ROUTING_KEY = 'awx_private_queue'
CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TRACK_STARTED = True
CELERYD_TASK_TIME_LIMIT = None
CELERYD_TASK_SOFT_TIME_LIMIT = None
CELERYD_POOL_RESTARTS = True
CELERYD_AUTOSCALER = 'awx.main.utils.autoscale:DynamicAutoScaler'
CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'
CELERY_IMPORTS = ('awx.main.scheduler.tasks',)
CELERY_QUEUES = ()
CELERY_ROUTES = ('awx.main.utils.ha.AWXCeleryRouter',)
def log_celery_failure(*args):
# Import annotations lazily to avoid polluting the `awx.settings` namespace
# and causing circular imports
from awx.main.tasks import log_celery_failure
return log_celery_failure(*args)
CELERY_ANNOTATIONS = {'*': {'on_failure': log_celery_failure}}
CELERYBEAT_SCHEDULER = 'celery.beat.PersistentScheduler'
CELERYBEAT_MAX_LOOP_INTERVAL = 60
CELERYBEAT_SCHEDULE = {
'tower_scheduler': {
'task': 'awx.main.tasks.awx_periodic_scheduler',
@@ -525,9 +494,6 @@ CELERYBEAT_SCHEDULE = {
}
AWX_INCONSISTENT_TASK_INTERVAL = 60 * 3
# Celery queues that will always be listened to by celery workers
# Note: Broadcast queues have unique, auto-generated names, with the alias
# property value of the original queue name.
AWX_CELERY_QUEUES_STATIC = [
six.text_type(CELERY_DEFAULT_QUEUE),
]
@@ -626,8 +592,8 @@ SOCIAL_AUTH_SAML_ENABLED_IDPS = {}
SOCIAL_AUTH_SAML_ORGANIZATION_ATTR = {}
SOCIAL_AUTH_SAML_TEAM_ATTR = {}
# Any ANSIBLE_* settings will be passed to the subprocess environment by the
# celery task.
# Any ANSIBLE_* settings will be passed to the task runner subprocess
# environment
# Do not want AWX to ask interactive questions and want it to be friendly with
# reprovisioning
@@ -641,8 +607,7 @@ ANSIBLE_PARAMIKO_RECORD_HOST_KEYS = False
# output
ANSIBLE_FORCE_COLOR = True
# Additional environment variables to be passed to the subprocess started by
# the celery task.
# Additional environment variables to be passed to the ansible subprocesses
AWX_TASK_ENV = {}
# Flag to enable/disable updating hosts M2M when saving job events.
@@ -1071,6 +1036,15 @@ LOGGING = {
'backupCount': 5,
'formatter':'simple',
},
'callback_receiver': {
'level': 'WARNING',
'class':'logging.handlers.RotatingFileHandler',
'filters': ['require_debug_false'],
'filename': os.path.join(LOG_ROOT, 'callback_receiver.log'),
'maxBytes': 1024 * 1024 * 5, # 5 MB
'backupCount': 5,
'formatter':'simple',
},
'dispatcher': {
'level': 'WARNING',
'class':'logging.handlers.RotatingFileHandler',
@@ -1080,6 +1054,10 @@ LOGGING = {
'backupCount': 5,
'formatter':'dispatcher',
},
'celery.beat': {
'class':'logging.StreamHandler',
'level': 'ERROR'
}, # don't log every celerybeat wakeup
'inventory_import': {
'level': 'DEBUG',
'class':'logging.StreamHandler',
@@ -1162,6 +1140,9 @@ LOGGING = {
'awx.main': {
'handlers': ['null']
},
'awx.main.commands.run_callback_receiver': {
'handlers': ['callback_receiver'],
},
'awx.main.dispatch': {
'handlers': ['dispatcher'],
},

16
awx/settings/development.py
View File

@@ -68,13 +68,6 @@ template['OPTIONS']['loaders'] = (
'django.template.loaders.app_directories.Loader',
)
# Disable capturing all SQL queries when running celeryd in development.
if 'celery' in sys.argv:
SQL_DEBUG = False
CELERYD_HIJACK_ROOT_LOGGER = False
CELERYD_LOG_COLOR = True
CALLBACK_QUEUE = "callback_tasks"
# Enable dynamically pulling roles from a requirement.yml file
@@ -149,15 +142,6 @@ except ImportError:
CLUSTER_HOST_ID = socket.gethostname()
# Supervisor service name dictionary used for programatic restart
SERVICE_NAME_DICT = {
"celery": "celery",
"callback": "receiver",
"runworker": "channels",
"uwsgi": "uwsgi",
"daphne": "daphne",
"nginx": "nginx"}
try:
socket.gethostbyname('docker.for.mac.host.internal')
os.environ['SDB_NOTIFY_HOST'] = 'docker.for.mac.host.internal'

7
awx/settings/local_settings.py.docker_compose
View File

@@ -73,13 +73,13 @@ if "pytest" in sys.modules:
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'awx.sqlite3'),
'TEST': {
# Test database cannot be :memory: for celery/inventory tests.
# Test database cannot be :memory: for inventory tests.
'NAME': os.path.join(BASE_DIR, 'awx_test.sqlite3'),
},
}
}
# Celery AMQP configuration.
# AMQP configuration.
BROKER_URL = "amqp://{}:{}@{}/{}".format(os.environ.get("RABBITMQ_USER"),
os.environ.get("RABBITMQ_PASS"),
os.environ.get("RABBITMQ_HOST"),
@@ -138,8 +138,7 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST']
# REMOTE_HOST_HEADERS will be trusted unconditionally')
PROXY_IP_WHITELIST = []
# Define additional environment variables to be passed to subprocess started by
# the celery task.
# Define additional environment variables to be passed to ansible subprocesses
#AWX_TASK_ENV['FOO'] = 'BAR'
# If set, use -vvv for project updates instead of -v for more output.

7
awx/settings/local_settings.py.example
View File

@@ -39,13 +39,13 @@ if is_testing(sys.argv):
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'awx.sqlite3'),
'TEST': {
# Test database cannot be :memory: for celery/inventory tests.
# Test database cannot be :memory: for tests.
'NAME': os.path.join(BASE_DIR, 'awx_test.sqlite3'),
},
}
}
# Celery AMQP configuration.
# AMQP configuration.
BROKER_URL = 'amqp://guest:guest@localhost:5672'
# Set True to enable additional logging from the job_event_callback plugin
@@ -94,8 +94,7 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST']
# REMOTE_HOST_HEADERS will be trusted unconditionally')
PROXY_IP_WHITELIST = []
# Define additional environment variables to be passed to subprocess started by
# the celery task.
# Define additional environment variables to be passed to ansible subprocesses
#AWX_TASK_ENV['FOO'] = 'BAR'
# If set, use -vvv for project updates instead of -v for more output.

10
awx/settings/production.py
View File

@@ -54,21 +54,13 @@ AWX_ISOLATED_USERNAME = 'awx'
LOGGING['handlers']['tower_warnings']['filename'] = '/var/log/tower/tower.log'
LOGGING['handlers']['callback_receiver']['filename'] = '/var/log/tower/callback_receiver.log'
LOGGING['handlers']['dispatcher']['filename'] = '/var/log/tower/dispatcher.log'
LOGGING['handlers']['task_system']['filename'] = '/var/log/tower/task_system.log'
LOGGING['handlers']['fact_receiver']['filename'] = '/var/log/tower/fact_receiver.log'
LOGGING['handlers']['management_playbooks']['filename'] = '/var/log/tower/management_playbooks.log'
LOGGING['handlers']['system_tracking_migrations']['filename'] = '/var/log/tower/tower_system_tracking_migrations.log'
LOGGING['handlers']['rbac_migrations']['filename'] = '/var/log/tower/tower_rbac_migrations.log'
# Supervisor service name dictionary used for programatic restart
SERVICE_NAME_DICT = {
"beat": "awx-celery-beat",
"celery": "awx-celery",
"callback": "awx-callback-receiver",
"channels": "awx-channels-worker",
"uwsgi": "awx-uwsgi",
"daphne": "awx-daphne"}
# Store a snapshot of default settings at this point before loading any
# customizable config files.
DEFAULTS_SNAPSHOT = {}