From 41fe9e1caf0d171f2d2da9c5be84332654a2c2e1 Mon Sep 17 00:00:00 2001
From: Wayne Witzel III <wayne@riotousliving.com>
Date: Fri, 18 May 2018 12:46:54 -0400
Subject: [PATCH 1/3] Check if the project update for the project we are trying
 to lock is canceled

---
 awx/main/tasks.py | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index 68dd21e804..79cc93bc51 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -5,6 +5,7 @@
 from collections import OrderedDict, namedtuple
 import ConfigParser
 import cStringIO
+import errno
 import functools
 import importlib
 import json
@@ -1668,18 +1669,28 @@ class RunProjectUpdate(BaseTask):
             logger.error(six.text_type("I/O error({0}) while trying to open lock file [{1}]: {2}").format(e.errno, lock_path, e.strerror))
             raise
 
-        try:
-            start_time = time.time()
-            fcntl.flock(self.lock_fd, fcntl.LOCK_EX)
-            waiting_time = time.time() - start_time
-            if waiting_time > 1.0:
-                logger.info(six.text_type(
-                    '{} spent {} waiting to acquire lock for local source tree '
-                    'for path {}.').format(instance.log_format, waiting_time, lock_path))
-        except IOError as e:
-            os.close(self.lock_fd)
-            logger.error(six.text_type("I/O error({0}) while trying to aquire lock on file [{1}]: {2}").format(e.errno, lock_path, e.strerror))
-            raise
+        start_time = time.time()
+        while True:
+            try:
+                instance.refresh_from_db()
+                if instance.cancel_flag:
+                    logger.info(six.text_type("ProjectUpdate({0}) was cancelled".format(instance.pk)))
+                    return
+                fcntl.flock(self.lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except IOError as e:
+                if e.errno != errno.EAGAIN:
+                    os.close(self.lock_fd)
+                    logger.error(six.text_type("I/O error({0}) while trying to aquire lock on file [{1}]: {2}").format(e.errno, lock_path, e.strerror))
+                    raise
+                else:
+                    time.sleep(1.0)
+        waiting_time = time.time() - start_time
+
+        if waiting_time > 1.0:
+            logger.info(six.text_type(
+                '{} spent {} waiting to acquire lock for local source tree '
+                'for path {}.').format(instance.log_format, waiting_time, lock_path))
 
     def pre_run_hook(self, instance, **kwargs):
         # re-create root project folder if a natural disaster has destroyed it

From 5279b102cba874cb533d1bf19a8362b31dceb401 Mon Sep 17 00:00:00 2001
From: Wayne Witzel III <wayne@riotousliving.com>
Date: Fri, 18 May 2018 13:52:51 -0400
Subject: [PATCH 2/3] Fix task unit test

---
 awx/main/tests/unit/test_tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py
index c4e3abe9b9..bc403bd015 100644
--- a/awx/main/tests/unit/test_tasks.py
+++ b/awx/main/tests/unit/test_tasks.py
@@ -2171,6 +2171,7 @@ def test_aquire_lock_acquisition_fail_logged(fcntl_flock, logging_getLogger, os_
 
     instance = mock.Mock()
     instance.get_lock_file.return_value = 'this_file_does_not_exist'
+    instance.cancel_flag = False
 
     os_open.return_value = 3
 
@@ -2180,7 +2181,6 @@ def test_aquire_lock_acquisition_fail_logged(fcntl_flock, logging_getLogger, os_
     fcntl_flock.side_effect = err
 
     ProjectUpdate = tasks.RunProjectUpdate()
-
     with pytest.raises(IOError, message='dummy message'):
         ProjectUpdate.acquire_lock(instance)
     os_close.assert_called_with(3)

From 9fe44cfaae0be7ee78987683b59deb3a38fff6a4 Mon Sep 17 00:00:00 2001
From: Wayne Witzel III <wayne@riotousliving.com>
Date: Fri, 18 May 2018 14:00:33 -0400
Subject: [PATCH 3/3] check EACCES and only refresh cancel_flag

---
 awx/main/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index 79cc93bc51..ad22a7d163 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -1672,14 +1672,14 @@ class RunProjectUpdate(BaseTask):
         start_time = time.time()
         while True:
             try:
-                instance.refresh_from_db()
+                instance.refresh_from_db(fields=['cancel_flag'])
                 if instance.cancel_flag:
                     logger.info(six.text_type("ProjectUpdate({0}) was cancelled".format(instance.pk)))
                     return
                 fcntl.flock(self.lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
                 break
             except IOError as e:
-                if e.errno != errno.EAGAIN:
+                if e.errno not in (errno.EAGAIN, errno.EACCES):
                     os.close(self.lock_fd)
                     logger.error(six.text_type("I/O error({0}) while trying to aquire lock on file [{1}]: {2}").format(e.errno, lock_path, e.strerror))
                     raise