External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-07-05 05:18:02 -02:30
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4275 from chrismeyersfsu/redis_throttle_reconnect

Browse Source 
exponential backoff on cb receiver reconnect
This commit is contained in:
Chris Meyers
2020-04-28 15:42:45 -04:00
committed by GitHub
parent 6b976c4239 a8f52c1639
commit cd15a5c082
1 changed files with 17 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
awx/main/dispatch/worker/base.py
View File

@@ -8,6 +8,7 @@ import sys
import redis import redis
import json import json
import psycopg2 import psycopg2
import time
from uuid import UUID from uuid import UUID
from queue import Empty as QueueEmpty from queue import Empty as QueueEmpty
@@ -116,14 +117,19 @@ class AWXConsumerRedis(AWXConsumerBase):
super(AWXConsumerRedis, self).run(*args, **kwargs) super(AWXConsumerRedis, self).run(*args, **kwargs)
self.worker.on_start() self.worker.on_start()
time_to_sleep = 1
while True:
queue = redis.Redis.from_url(settings.BROKER_URL) queue = redis.Redis.from_url(settings.BROKER_URL)
while True: while True:
try: try:
res = queue.blpop(self.queues) res = queue.blpop(self.queues)
time_to_sleep = 1
res = json.loads(res[1]) res = json.loads(res[1])
self.process_task(res) self.process_task(res)
except redis.exceptions.RedisError: except redis.exceptions.RedisError:
logger.exception("encountered an error communicating with redis") time_to_sleep = min(time_to_sleep*2, 30)
logger.exception(f"encountered an error communicating with redis. Reconnect attempt in {time_to_sleep} seconds")
time.sleep(time_to_sleep)
except (json.JSONDecodeError, KeyError): except (json.JSONDecodeError, KeyError):
logger.exception("failed to decode JSON message from redis") logger.exception("failed to decode JSON message from redis")
if self.should_stop: if self.should_stop: