Handle database errors and retry

awx/main/tasks.py

@@ -27,7 +27,7 @@ from celery import Task, task
 
 # Django
 from django.conf import settings
-from django.db import transaction
+from django.db import transaction, DatabaseError
 from django.utils.datastructures import SortedDict
 from django.utils.timezone import now
 
@@ -87,6 +87,9 @@ class BaseTask(Task):
         # Commit outstanding transaction so that we fetch the latest object
         # from the database.
         transaction.commit()
+        save_succeeded = True
+        while True:
+            try:
                 instance = self.model.objects.get(pk=pk)
                 if updates:
                     update_fields = ['modified']
@@ -100,6 +103,13 @@ class BaseTask(Task):
                             update_fields.append('failed')
                     instance.save(update_fields=update_fields)
                     transaction.commit()
+                    save_succeeded = True
+            except DatabaseError as e:
+                logger.debug("Database error encountered, retrying: " + str(e))
+                save_succeeded = False
+            finally:
+                if save_succeeded:
+                    break
         return instance
 
     def get_model(self, pk):