diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index bf60e5b77c..a49046e0eb 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -208,6 +208,12 @@ class ProjectOptions(models.Model): results.append(smart_text(playbook)) return sorted(results, key=lambda x: smart_str(x).lower()) + def get_lock_file(self): + proj_path = self.get_project_path() + if not proj_path: + return None + return proj_path + '.lock' + class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin): ''' diff --git a/awx/main/tasks.py b/awx/main/tasks.py index aa5b9a0f0f..8c767285ee 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -23,6 +23,7 @@ import uuid from distutils.version import LooseVersion as Version from datetime import timedelta import yaml +import fcntl try: import psutil except: @@ -1328,7 +1329,46 @@ class RunProjectUpdate(BaseTask): instance_actual.save() return OutputEventFilter(stdout_handle, raw_callback=raw_callback) + def release_lock(self, instance): + try: + fcntl.flock(self.lock_fd, fcntl.LOCK_UN) + except IOError as e: + logger.error("I/O error({0}) while trying to open lock file [{1}]: {2}".format(e.errno, instance.get_lock_file(), e.strerror)) + os.close(self.lock_fd) + raise + + os.close(self.lock_fd) + self.lock_fd = None + + ''' + Note: We don't support blocking=False + ''' + def acquire_lock(self, instance, blocking=True): + lock_path = instance.get_lock_file() + if lock_path is None: + raise RuntimeError(u'Invalid lock file path') + + try: + self.lock_fd = os.open(lock_path, os.O_RDONLY | os.O_CREAT) + except OSError as e: + logger.error("I/O error({0}) while trying to open lock file [{1}]: {2}".format(e.errno, lock_path, e.strerror)) + raise + + try: + fcntl.flock(self.lock_fd, fcntl.LOCK_EX) + except IOError as e: + os.close(self.lock_fd) + logger.error("I/O error({0}) while trying to aquire lock on file [{1}]: {2}".format(e.errno, lock_path, e.strerror)) + raise + + def pre_run_hook(self, instance, **kwargs): + if instance.launch_type == 'sync': + self.acquire_lock(instance) + def post_run_hook(self, instance, status, **kwargs): + if instance.launch_type == 'sync': + self.release_lock(instance) + if instance.job_type == 'check' and status not in ('failed', 'canceled',): p = instance.project fd = open(self.revision_path, 'r') diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 16b9bc6b14..6465512da0 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1,6 +1,9 @@ from contextlib import contextmanager +import os +import fcntl import pytest +import mock import yaml from awx.main.models import ( @@ -143,3 +146,63 @@ def test_openstack_client_config_generation_with_private_source_vars(mocker, sou 'private': expected } } + + +def test_os_open_oserror(): + with pytest.raises(OSError): + os.open('this_file_does_not_exist', os.O_RDONLY) + + +def test_fcntl_ioerror(): + with pytest.raises(IOError): + fcntl.flock(99999, fcntl.LOCK_EX) + + +@mock.patch('os.open') +@mock.patch('logging.getLogger') +def test_aquire_lock_open_fail_logged(logging_getLogger, os_open): + err = OSError() + err.errno = 3 + err.strerror = 'dummy message' + + instance = mock.Mock() + instance.get_lock_file.return_value = 'this_file_does_not_exist' + + os_open.side_effect = err + + logger = mock.Mock() + logging_getLogger.return_value = logger + + ProjectUpdate = tasks.RunProjectUpdate() + + with pytest.raises(OSError, errno=3, strerror='dummy message'): + ProjectUpdate.acquire_lock(instance) + assert logger.err.called_with("I/O error({0}) while trying to open lock file [{1}]: {2}".format(3, 'this_file_does_not_exist', 'dummy message')) + + +@mock.patch('os.open') +@mock.patch('os.close') +@mock.patch('logging.getLogger') +@mock.patch('fcntl.flock') +def test_aquire_lock_acquisition_fail_logged(fcntl_flock, logging_getLogger, os_close, os_open): + err = IOError() + err.errno = 3 + err.strerror = 'dummy message' + + instance = mock.Mock() + instance.get_lock_file.return_value = 'this_file_does_not_exist' + + os_open.return_value = 3 + + logger = mock.Mock() + logging_getLogger.return_value = logger + + fcntl_flock.side_effect = err + + ProjectUpdate = tasks.RunProjectUpdate() + + with pytest.raises(IOError, errno=3, strerror='dummy message'): + ProjectUpdate.acquire_lock(instance) + os_close.assert_called_with(3) + assert logger.err.called_with("I/O error({0}) while trying to aquire lock on file [{1}]: {2}".format(3, 'this_file_does_not_exist', 'dummy message')) +