From 0e97dc4b84a6e677b1b44be17c578fce69a8c1a7 Mon Sep 17 00:00:00 2001
From: Chris Meyers <chris.meyers.fsu@gmail.com>
Date: Tue, 10 Oct 2017 12:34:49 -0400
Subject: [PATCH] Beat and celery clustering fixes

* use embedded beat rather than standalone
* dynamically set celeryd hostname at runtime
* add embeded beat flag to celery startup
* Embedded beat mode routes will piggyback off of celery worker setup
signal
---
 Makefile                                         |  2 +-
 awx/main/tasks.py                                | 10 ++++++++--
 installer/image_build/files/supervisor_task.conf | 14 +-------------
 tools/docker-compose/supervisor.conf             |  2 +-
 4 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/Makefile b/Makefile
index 874a4e52ee..99e4d0c325 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ COMPOSE_HOST ?= $(shell hostname)
 
 VENV_BASE ?= /venv
 SCL_PREFIX ?=
-CELERY_SCHEDULE_FILE ?= /celerybeat-schedule
+CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db
 
 DEV_DOCKER_TAG_BASE ?= gcr.io/ansible-tower-engineering
 # Python packages to install only from source (not from binary wheels)
diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index 36274b926d..0b52f2e93e 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -26,7 +26,7 @@ except Exception:
 
 # Celery
 from celery import Task, shared_task
-from celery.signals import celeryd_init, worker_process_init, worker_shutdown, worker_ready, beat_init
+from celery.signals import celeryd_init, worker_process_init, worker_shutdown, worker_ready, celeryd_after_setup
 
 # Django
 from django.conf import settings
@@ -168,7 +168,6 @@ def handle_ha_toplogy_worker_ready(sender, **kwargs):
                 .format(instance.hostname, removed_queues, added_queues))
 
 
-@beat_init.connect
 @celeryd_init.connect
 def handle_update_celery_routes(sender=None, conf=None, **kwargs):
     conf = conf if conf else sender.app.conf
@@ -179,6 +178,13 @@ def handle_update_celery_routes(sender=None, conf=None, **kwargs):
                 .format(instance.hostname, added_routes, conf.CELERY_ROUTES))
 
 
+@celeryd_after_setup.connect
+def handle_update_celery_hostname(sender, instance, **kwargs):
+    tower_instance = Instance.objects.me()
+    instance.hostname = 'celery@{}'.format(tower_instance.hostname)
+    logger.warn("Set hostname to {}".format(instance.hostname))
+
+
 @shared_task(queue='tower', base=LogErrorsTask)
 def send_notifications(notification_list, job_id=None):
     if not isinstance(notification_list, list):
diff --git a/installer/image_build/files/supervisor_task.conf b/installer/image_build/files/supervisor_task.conf
index 19aac3c3b0..3bc71cf75b 100644
--- a/installer/image_build/files/supervisor_task.conf
+++ b/installer/image_build/files/supervisor_task.conf
@@ -3,7 +3,7 @@ nodaemon = True
 umask = 022
 
 [program:celery]
-command = /var/lib/awx/venv/awx/bin/celery worker -A awx -l debug --autoscale=4 -Ofair -Q tower_broadcast_all -n celery@%(ENV_HOSTNAME)s
+command = /var/lib/awx/venv/awx/bin/celery worker -A awx -B -l debug --autoscale=4 -Ofair -s /var/lib/awx/beat.db -Q tower_broadcast_all -n celery@$(ENV_HOSTNAME)s
 directory = /var/lib/awx
 environment = LANGUAGE="en_US.UTF-8",LANG="en_US.UTF-8",LC_ALL="en_US.UTF-8",LC_CTYPE="en_US.UTF-8"
 #user = {{ aw_user }}
@@ -15,18 +15,6 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
-[program:awx-celeryd-beat]
-command = /var/lib/awx/venv/awx/bin/celery beat -A awx -l debug --pidfile= -s /var/lib/awx/beat.db
-directory = /var/lib/awx
-autostart = true
-autorestart = true
-stopwaitsecs = 5
-redirect_stderr=true
-stdout_logfile = /dev/stdout
-stdout_logfile_maxbytes = 0
-stderr_logfile = /dev/stderr
-stderr_logfile_maxbytes = 0
-
 [program:callback-receiver]
 command = awx-manage run_callback_receiver
 directory = /var/lib/awx
diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf
index b0700e1442..cedb784324 100644
--- a/tools/docker-compose/supervisor.conf
+++ b/tools/docker-compose/supervisor.conf
@@ -4,7 +4,7 @@ minfds = 4096
 nodaemon=true
 
 [program:celeryd]
-command = celery worker -A awx -l DEBUG -B -Ofair --autoscale=100,4 --schedule=/celerybeat-schedule -Q tower_broadcast_all -n celery@%(ENV_HOSTNAME)s
+command = celery worker -A awx -l DEBUG -B --autoscale=20,3 -Ofair -s /var/lib/awx/beat.db -Q tower_broadcast_all -n celery@%(ENV_HOSTNAME)s
 autostart = true
 autorestart = true
 redirect_stderr=true