sleep backoff on cb receiver reconnect

* Sleep before trying to reconnect
Most common reason for entering this reconnect loop is when Redis
service stops before the callback receiver when stopping tower services.

awx/main/dispatch/worker/base.py

@@ -8,6 +8,7 @@ import sys
 import redis
 import json
 import psycopg2
+import time
 from uuid import UUID
 from queue import Empty as QueueEmpty
 
@@ -116,18 +117,23 @@ class AWXConsumerRedis(AWXConsumerBase):
         super(AWXConsumerRedis, self).run(*args, **kwargs)
         self.worker.on_start()
 
-        queue = redis.Redis.from_url(settings.BROKER_URL)
+        retry = 0
         while True:
-            try:
+            queue = redis.Redis.from_url(settings.BROKER_URL)
-                res = queue.blpop(self.queues)
+            while True:
-                res = json.loads(res[1])
+                try:
-                self.process_task(res)
+                    res = queue.blpop(self.queues)
-            except redis.exceptions.RedisError:
+                    retry = 0
-                logger.exception("encountered an error communicating with redis")
+                    res = json.loads(res[1])
-            except (json.JSONDecodeError, KeyError):
+                    self.process_task(res)
-                logger.exception("failed to decode JSON message from redis")
+                except redis.exceptions.RedisError:
-            if self.should_stop:
+                    logger.exception(f"encountered an error communicating with redis. Reconnect attempt {retry}")
-                return
+                    retry += 1
+                    time.sleep(min(retry * 2, 30))
+                except (json.JSONDecodeError, KeyError):
+                    logger.exception("failed to decode JSON message from redis")
+                if self.should_stop:
+                    return
 
 
 class AWXConsumerPG(AWXConsumerBase):