From f1b1c4ee97ee993ca6cdff529e4328eac6fcd170 Mon Sep 17 00:00:00 2001
From: Chris Meyers <chris.meyers.fsu@gmail.com>
Date: Fri, 30 Jun 2017 16:09:49 -0400
Subject: [PATCH] avoid instance record deadlock by using pglock

---
 awx/main/scheduler/__init__.py   | 18 +++++++++---------
 docs/licenses/django-pglocks.txt |  3 +++
 requirements/requirements.in     |  1 +
 requirements/requirements.txt    |  1 +
 4 files changed, 14 insertions(+), 9 deletions(-)
 create mode 100644 docs/licenses/django-pglocks.txt

diff --git a/awx/main/scheduler/__init__.py b/awx/main/scheduler/__init__.py
index 383b71a552..2136a58fe8 100644
--- a/awx/main/scheduler/__init__.py
+++ b/awx/main/scheduler/__init__.py
@@ -9,7 +9,6 @@ from sets import Set
 # Django
 from django.conf import settings
 from django.db import transaction, connection
-from django.db.utils import DatabaseError
 from django.utils.translation import ugettext_lazy as _
 from django.utils.timezone import now as tz_now
 
@@ -24,6 +23,8 @@ from awx.main.tasks import _send_notification_templates
 # Celery
 from celery.task.control import inspect
 
+from django_pglocks import advisory_lock
+
 logger = logging.getLogger('awx.main.scheduler')
 
 
@@ -459,13 +460,12 @@ class TaskManager():
         logger.debug("Starting Schedule")
         with transaction.atomic():
             # Lock
-            try:
-                Instance.objects.select_for_update(nowait=True).all()[0]
-            except DatabaseError:
-                return
+            with advisory_lock('task_manager_lock', wait=False) as acquired:
+                if acquired is False:
+                    return
 
-            finished_wfjs = self._schedule()
+                finished_wfjs = self._schedule()
 
-        # Operations whose queries rely on modifications made during the atomic scheduling session
-        for wfj in WorkflowJob.objects.filter(id__in=finished_wfjs):
-            _send_notification_templates(wfj, 'succeeded' if wfj.status == 'successful' else 'failed')
+                # Operations whose queries rely on modifications made during the atomic scheduling session
+                for wfj in WorkflowJob.objects.filter(id__in=finished_wfjs):
+                    _send_notification_templates(wfj, 'succeeded' if wfj.status == 'successful' else 'failed')
diff --git a/docs/licenses/django-pglocks.txt b/docs/licenses/django-pglocks.txt
new file mode 100644
index 0000000000..3b74730882
--- /dev/null
+++ b/docs/licenses/django-pglocks.txt
@@ -0,0 +1,3 @@
+Copyright (c) 2013 Christophe Pettus
+
+Licensed under the MIT License.
diff --git a/requirements/requirements.in b/requirements/requirements.in
index a178a9e8a1..82ed3156c0 100644
--- a/requirements/requirements.in
+++ b/requirements/requirements.in
@@ -16,6 +16,7 @@ django-crum==0.7.1
 django-extensions==1.7.8
 django-jsonfield==1.0.1
 django-polymorphic==1.2
+django-pglocks==1.0.2
 django-radius==1.1.0
 django-solo==1.1.2
 django-split-settings==0.2.5
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index b2d3ea2dea..7913a9e971 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -59,6 +59,7 @@ django-crum==0.7.1
 django-extensions==1.7.8
 django-jsonfield==1.0.1
 django-polymorphic==1.2
+django-pglocks==1.0.2
 django-radius==1.1.0
 django-solo==1.1.2
 django-split-settings==0.2.5