Handle database errors and retry

This commit is contained in:
Matthew Jones
2014-01-30 14:00:49 -05:00
parent a85e109da7
commit b226737ec5

View File

@@ -27,7 +27,7 @@ from celery import Task, task
# Django # Django
from django.conf import settings from django.conf import settings
from django.db import transaction from django.db import transaction, DatabaseError
from django.utils.datastructures import SortedDict from django.utils.datastructures import SortedDict
from django.utils.timezone import now from django.utils.timezone import now
@@ -87,19 +87,29 @@ class BaseTask(Task):
# Commit outstanding transaction so that we fetch the latest object # Commit outstanding transaction so that we fetch the latest object
# from the database. # from the database.
transaction.commit() transaction.commit()
instance = self.model.objects.get(pk=pk) save_succeeded = True
if updates: while True:
update_fields = ['modified'] try:
for field, value in updates.items(): instance = self.model.objects.get(pk=pk)
if field in ('result_stdout', 'result_traceback'): if updates:
for srch, repl in output_replacements: update_fields = ['modified']
value = value.replace(srch, repl) for field, value in updates.items():
setattr(instance, field, value) if field in ('result_stdout', 'result_traceback'):
update_fields.append(field) for srch, repl in output_replacements:
if field == 'status': value = value.replace(srch, repl)
update_fields.append('failed') setattr(instance, field, value)
instance.save(update_fields=update_fields) update_fields.append(field)
transaction.commit() if field == 'status':
update_fields.append('failed')
instance.save(update_fields=update_fields)
transaction.commit()
save_succeeded = True
except DatabaseError as e:
logger.debug("Database error encountered, retrying: " + str(e))
save_succeeded = False
finally:
if save_succeeded:
break
return instance return instance
def get_model(self, pk): def get_model(self, pk):