avoid instance record deadlock by using pglock

awx/main/scheduler/__init__.py

@@ -9,7 +9,6 @@ from sets import Set
 # Django
 from django.conf import settings
 from django.db import transaction, connection
-from django.db.utils import DatabaseError
 from django.utils.translation import ugettext_lazy as _
 from django.utils.timezone import now as tz_now
 
@@ -24,6 +23,8 @@ from awx.main.tasks import _send_notification_templates
 # Celery
 from celery.task.control import inspect
 
+from django_pglocks import advisory_lock
+
 logger = logging.getLogger('awx.main.scheduler')
 
 
@@ -459,13 +460,12 @@ class TaskManager():
         logger.debug("Starting Schedule")
         with transaction.atomic():
             # Lock
-            try:
-                Instance.objects.select_for_update(nowait=True).all()[0]
-            except DatabaseError:
-                return
+            with advisory_lock('task_manager_lock', wait=False) as acquired:
+                if acquired is False:
+                    return
 
-            finished_wfjs = self._schedule()
+                finished_wfjs = self._schedule()
 
-        # Operations whose queries rely on modifications made during the atomic scheduling session
-        for wfj in WorkflowJob.objects.filter(id__in=finished_wfjs):
-            _send_notification_templates(wfj, 'succeeded' if wfj.status == 'successful' else 'failed')
+                # Operations whose queries rely on modifications made during the atomic scheduling session
+                for wfj in WorkflowJob.objects.filter(id__in=finished_wfjs):
+                    _send_notification_templates(wfj, 'succeeded' if wfj.status == 'successful' else 'failed')

docs/licenses/django-pglocks.txt

@@ -0,0 +1,3 @@
+Copyright (c) 2013 Christophe Pettus
+
+Licensed under the MIT License.

requirements/requirements.in

@@ -16,6 +16,7 @@ django-crum==0.7.1
 django-extensions==1.7.8
 django-jsonfield==1.0.1
 django-polymorphic==1.2
+django-pglocks==1.0.2
 django-radius==1.1.0
 django-solo==1.1.2
 django-split-settings==0.2.5

requirements/requirements.txt

@@ -59,6 +59,7 @@ django-crum==0.7.1
 django-extensions==1.7.8
 django-jsonfield==1.0.1
 django-polymorphic==1.2
+django-pglocks==1.0.2
 django-radius==1.1.0
 django-solo==1.1.2
 django-split-settings==0.2.5