Merge pull request #6002 from chrismeyersfsu/fix-5984

lock projects on project sync

awx/main/models/projects.py

@@ -208,6 +208,12 @@ class ProjectOptions(models.Model):
                     results.append(smart_text(playbook))
         return sorted(results, key=lambda x: smart_str(x).lower())
 
+    def get_lock_file(self):
+        proj_path = self.get_project_path()
+        if not proj_path:
+            return None
+        return proj_path + '.lock'
+
 
 class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin):
     '''

awx/main/tasks.py

@@ -23,6 +23,7 @@ import uuid
 from distutils.version import LooseVersion as Version
 from datetime import timedelta
 import yaml
+import fcntl
 try:
     import psutil
 except:
@@ -1328,7 +1329,46 @@ class RunProjectUpdate(BaseTask):
             instance_actual.save()
         return OutputEventFilter(stdout_handle, raw_callback=raw_callback)
 
+    def release_lock(self, instance):
+        try:
+            fcntl.flock(self.lock_fd, fcntl.LOCK_UN)
+        except IOError as e:
+            logger.error("I/O error({0}) while trying to open lock file [{1}]: {2}".format(e.errno, instance.get_lock_file(), e.strerror))
+            os.close(self.lock_fd)
+            raise
+
+        os.close(self.lock_fd)
+        self.lock_fd = None
+
+    '''
+    Note: We don't support blocking=False
+    '''
+    def acquire_lock(self, instance, blocking=True):
+        lock_path = instance.get_lock_file()
+        if lock_path is None:
+            raise RuntimeError(u'Invalid lock file path')
+
+        try:
+            self.lock_fd = os.open(lock_path, os.O_RDONLY | os.O_CREAT)
+        except OSError as e:
+            logger.error("I/O error({0}) while trying to open lock file [{1}]: {2}".format(e.errno, lock_path, e.strerror))
+            raise
+
+        try:
+            fcntl.flock(self.lock_fd, fcntl.LOCK_EX)
+        except IOError as e:
+            os.close(self.lock_fd)
+            logger.error("I/O error({0}) while trying to aquire lock on file [{1}]: {2}".format(e.errno, lock_path, e.strerror))
+            raise
+    
+    def pre_run_hook(self, instance, **kwargs):
+        if instance.launch_type == 'sync':
+            self.acquire_lock(instance)
+
     def post_run_hook(self, instance, status, **kwargs):
+        if instance.launch_type == 'sync':
+            self.release_lock(instance)
+
         if instance.job_type == 'check' and status not in ('failed', 'canceled',):
             p = instance.project
             fd = open(self.revision_path, 'r')

awx/main/tests/unit/test_tasks.py

@@ -1,6 +1,9 @@
 from contextlib import contextmanager
 
+import os
+import fcntl
 import pytest
+import mock
 import yaml
 
 from awx.main.models import (
@@ -143,3 +146,63 @@ def test_openstack_client_config_generation_with_private_source_vars(mocker, sou
             'private': expected
         }
     }
+
+
+def test_os_open_oserror():
+    with pytest.raises(OSError):
+        os.open('this_file_does_not_exist', os.O_RDONLY)
+
+
+def test_fcntl_ioerror():
+    with pytest.raises(IOError):
+        fcntl.flock(99999, fcntl.LOCK_EX)
+
+
+@mock.patch('os.open')
+@mock.patch('logging.getLogger')
+def test_aquire_lock_open_fail_logged(logging_getLogger, os_open):
+    err = OSError()
+    err.errno = 3
+    err.strerror = 'dummy message'
+
+    instance = mock.Mock()
+    instance.get_lock_file.return_value = 'this_file_does_not_exist'
+
+    os_open.side_effect = err
+
+    logger = mock.Mock()
+    logging_getLogger.return_value = logger
+    
+    ProjectUpdate = tasks.RunProjectUpdate()
+
+    with pytest.raises(OSError, errno=3, strerror='dummy message'):
+        ProjectUpdate.acquire_lock(instance)
+    assert logger.err.called_with("I/O error({0}) while trying to open lock file [{1}]: {2}".format(3, 'this_file_does_not_exist', 'dummy message'))
+
+
+@mock.patch('os.open')
+@mock.patch('os.close')
+@mock.patch('logging.getLogger')
+@mock.patch('fcntl.flock')
+def test_aquire_lock_acquisition_fail_logged(fcntl_flock, logging_getLogger, os_close, os_open):
+    err = IOError()
+    err.errno = 3
+    err.strerror = 'dummy message'
+
+    instance = mock.Mock()
+    instance.get_lock_file.return_value = 'this_file_does_not_exist'
+
+    os_open.return_value = 3
+
+    logger = mock.Mock()
+    logging_getLogger.return_value = logger
+
+    fcntl_flock.side_effect = err
+    
+    ProjectUpdate = tasks.RunProjectUpdate()
+
+    with pytest.raises(IOError, errno=3, strerror='dummy message'):
+        ProjectUpdate.acquire_lock(instance)
+    os_close.assert_called_with(3)
+    assert logger.err.called_with("I/O error({0}) while trying to aquire lock on file [{1}]: {2}".format(3, 'this_file_does_not_exist', 'dummy message'))
+