replace celery task decorators with a kombu-based publisher

this commit implements the bulk of `awx-manage run_dispatcher`, a new
command that binds to RabbitMQ via kombu and balances messages across
a pool of workers that are similar to celeryd workers in spirit.
Specifically, this includes:

- a new decorator, `awx.main.dispatch.task`, which can be used to
  decorate functions or classes so that they can be designated as
  "Tasks"
- support for fanout/broadcast tasks (at this point in time, only
  `conf.Setting` memcached flushes use this functionality)
- support for job reaping
- support for success/failure hooks for job runs (i.e.,
  `handle_work_success` and `handle_work_error`)
- support for auto scaling worker pool that scale processes up and down
  on demand
- minimal support for RPC, such as status checks and pool recycle/reload
This commit is contained in:
Ryan Petrello
2018-08-08 13:41:07 -04:00
parent da74f1d01f
commit ff1e8cc356
54 changed files with 1606 additions and 1147 deletions

View File

@@ -3,8 +3,8 @@ umask = 022
minfds = 4096
nodaemon=true
[program:celeryd]
command = celery worker -A awx -l DEBUG -B -Ofair --autoscale=100,4 -s /var/lib/awx/beat.db --pidfile /tmp/celery_pid
[program:dispatcher]
command = awx-manage run_dispatcher
autostart = true
autorestart = true
redirect_stderr=true
@@ -51,14 +51,6 @@ redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:flower]
command = make flower
autostart = true
autorestart = true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:jupyter]
command = make jupyter
autostart = true
@@ -68,7 +60,7 @@ stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[group:awx-processes]
programs=celeryd,receiver,runworker,uwsgi,daphne,nginx,flower
programs=dispatcher,receiver,runworker,uwsgi,daphne,nginx
priority=5
[unix_http_server]