From 9b1961f60ded8786e55b555257ad51e4272882a2 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 26 Jul 2018 16:16:49 -0400 Subject: [PATCH] some celery-watcher tweaks * give celery more time to reply over AMQP (the default timeout is 1s) * record datestamps in celery watcher logs for debugging purposes --- awx/main/management/commands/watch_celery.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/awx/main/management/commands/watch_celery.py b/awx/main/management/commands/watch_celery.py index 6da5bef2f4..bd45f06803 100644 --- a/awx/main/management/commands/watch_celery.py +++ b/awx/main/management/commands/watch_celery.py @@ -1,3 +1,4 @@ +import datetime import os import signal import subprocess @@ -33,19 +34,25 @@ class Command(BaseCommand): INTERVAL = 60 + def _log(self, msg): + sys.stderr.write(datetime.datetime.utcnow().isoformat()) + sys.stderr.write(' ') + sys.stderr.write(msg) + sys.stderr.write('\n') + def handle(self, **options): app = Celery('awx') app.config_from_object('django.conf:settings') while True: try: - pongs = app.control.ping(['celery@{}'.format(settings.CLUSTER_HOST_ID)]) + pongs = app.control.ping(['celery@{}'.format(settings.CLUSTER_HOST_ID)], timeout=30) except Exception: pongs = [] if not pongs: - sys.stderr.write('celery is not responsive to ping over local AMQP\n') + self._log('celery is not responsive to ping over local AMQP') pid = self.getpid() if pid: - sys.stderr.write('sending SIGHUP to {}\n'.format(pid)) + self._log('sending SIGHUP to {}'.format(pid)) os.kill(pid, signal.SIGHUP) time.sleep(self.INTERVAL) @@ -56,4 +63,4 @@ class Command(BaseCommand): try: return int(subprocess.check_output(cmd, shell=True)) except Exception: - sys.stderr.write('could not detect celery pid\n') + self._log('could not detect celery pid')