From 80147acc1ce8a2292c1c4278335dcb1508ea8b73 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 9 Apr 2020 15:38:03 -0400 Subject: [PATCH] work around redis connection failures in the callback receiver if redis stops/starts, sometimes the callback receiver doesn't recover without a restart; this fixes that --- awx/main/dispatch/worker/base.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/awx/main/dispatch/worker/base.py b/awx/main/dispatch/worker/base.py index 40601adf62..f3d46e9e12 100644 --- a/awx/main/dispatch/worker/base.py +++ b/awx/main/dispatch/worker/base.py @@ -118,9 +118,14 @@ class AWXConsumerRedis(AWXConsumerBase): queue = redis.Redis.from_url(settings.BROKER_URL) while True: - res = queue.blpop(self.queues) - res = json.loads(res[1]) - self.process_task(res) + try: + res = queue.blpop(self.queues) + res = json.loads(res[1]) + self.process_task(res) + except redis.exceptions.RedisError: + logger.exception(f"encountered an error communicating with redis") + except (json.JSONDecodeError, KeyError): + logger.exception(f"failed to decode JSON message from redis") if self.should_stop: return