add additional DB retry logic to the callback receiver

initially, I implemented this for _only_ the task worker, but it's
probably needed for callback event workers, too
This commit is contained in:
Ryan Petrello
2018-11-29 10:44:37 -05:00
parent 273d7a83f2
commit 0391dbc292
3 changed files with 6 additions and 9 deletions

View File

@@ -1,7 +1,5 @@
import logging
import time
import os
import signal
import traceback
from django.conf import settings
@@ -110,8 +108,7 @@ class CallbackBrokerWorker(BaseWorker):
break
except (OperationalError, InterfaceError, InternalError):
if retries >= self.MAX_RETRIES:
logger.exception('Worker could not re-establish database connectivity, shutting down gracefully: Job {}'.format(job_identifier))
os.kill(os.getppid(), signal.SIGINT)
logger.exception('Worker could not re-establish database connectivity, giving up on event for Job {}'.format(job_identifier))
return
delay = 60 * retries
logger.exception('Database Error Saving Job Event, retry #{i} in {delay} seconds:'.format(