diff --git a/awx/__init__.py b/awx/__init__.py index eae7df87bd..62925baeb1 100644 --- a/awx/__init__.py +++ b/awx/__init__.py @@ -78,9 +78,10 @@ def oauth2_getattribute(self, attr): # Custom method to override # oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__ from django.conf import settings + from oauth2_provider.settings import DEFAULTS val = None - if 'migrate' not in sys.argv: + if (isinstance(attr, str)) and (attr in DEFAULTS) and (not attr.startswith('_')): # certain Django OAuth Toolkit migrations actually reference # setting lookups for references to model classes (e.g., # oauth2_settings.REFRESH_TOKEN_MODEL) diff --git a/awx/conf/settings.py b/awx/conf/settings.py index 36b7db7d49..9853d1bb01 100644 --- a/awx/conf/settings.py +++ b/awx/conf/settings.py @@ -1,7 +1,6 @@ # Python import contextlib import logging -import sys import threading import time import os @@ -31,7 +30,7 @@ from awx.conf.models import Setting logger = logging.getLogger('awx.conf.settings') -SETTING_MEMORY_TTL = 5 if 'callback_receiver' in ' '.join(sys.argv) else 0 +SETTING_MEMORY_TTL = 5 # Store a special value to indicate when a setting is not set in the database. SETTING_CACHE_NOTSET = '___notset___' @@ -403,11 +402,15 @@ class SettingsWrapper(UserSettingsHolder): key=lambda *args, **kwargs: SettingsWrapper.hashkey(*args, **kwargs), lock=lambda self: self.__dict__['_awx_conf_memoizedcache_lock'], ) + def _get_local_with_cache(self, name): + """Get value while accepting the in-memory cache if key is available""" + with _ctit_db_wrapper(trans_safe=True): + return self._get_local(name) + def __getattr__(self, name): value = empty if name in self.all_supported_settings: - with _ctit_db_wrapper(trans_safe=True): - value = self._get_local(name) + value = self._get_local_with_cache(name) if value is not empty: return value return self._get_default(name) diff --git a/awx/conf/signals.py b/awx/conf/signals.py index 843900d9e6..d8297becb4 100644 --- a/awx/conf/signals.py +++ b/awx/conf/signals.py @@ -28,6 +28,9 @@ def handle_setting_change(key, for_delete=False): cache_keys = {Setting.get_cache_key(k) for k in setting_keys} cache.delete_many(cache_keys) + # if we have changed a setting, we want to avoid mucking with the in-memory cache entirely + settings._awx_conf_memoizedcache.clear() + # Send setting_changed signal with new value for each setting. for setting_key in setting_keys: setting_changed.send(sender=Setting, setting=setting_key, value=getattr(settings, setting_key, None), enter=not bool(for_delete)) diff --git a/awx/conf/tests/unit/test_settings.py b/awx/conf/tests/unit/test_settings.py index a184fa3191..368b40660b 100644 --- a/awx/conf/tests/unit/test_settings.py +++ b/awx/conf/tests/unit/test_settings.py @@ -8,6 +8,8 @@ import codecs from uuid import uuid4 import time +from unittest import mock + from django.conf import LazySettings from django.core.cache.backends.locmem import LocMemCache from django.core.exceptions import ImproperlyConfigured @@ -299,3 +301,33 @@ def test_readonly_sensitive_cache_data_is_encrypted(settings): cache.set('AWX_ENCRYPTED', 'SECRET!') assert cache.get('AWX_ENCRYPTED') == 'SECRET!' assert native_cache.get('AWX_ENCRYPTED') == 'FRPERG!' + + +@pytest.mark.defined_in_file(AWX_VAR='DEFAULT') +def test_in_memory_cache_only_for_registered_settings(settings): + "Test that we only make use of the in-memory TTL cache for registered settings" + settings._awx_conf_memoizedcache.clear() + settings.MIDDLEWARE + assert len(settings._awx_conf_memoizedcache) == 0 # does not cache MIDDLEWARE + settings.registry.register('AWX_VAR', field_class=fields.CharField, category=_('System'), category_slug='system') + settings._wrapped.__dict__['all_supported_settings'] = ['AWX_VAR'] # because it is cached_property + settings._awx_conf_memoizedcache.clear() + assert settings.AWX_VAR == 'DEFAULT' + assert len(settings._awx_conf_memoizedcache) == 1 # caches registered settings + + +@pytest.mark.defined_in_file(AWX_VAR='DEFAULT') +def test_in_memory_cache_works(settings): + settings._awx_conf_memoizedcache.clear() + settings.registry.register('AWX_VAR', field_class=fields.CharField, category=_('System'), category_slug='system') + settings._wrapped.__dict__['all_supported_settings'] = ['AWX_VAR'] + + settings._awx_conf_memoizedcache.clear() + + with mock.patch('awx.conf.settings.SettingsWrapper._get_local', return_value='DEFAULT') as mock_get: + assert settings.AWX_VAR == 'DEFAULT' + mock_get.assert_called_once_with('AWX_VAR') + + with mock.patch.object(settings, '_get_local') as mock_get: + assert settings.AWX_VAR == 'DEFAULT' + mock_get.assert_not_called() diff --git a/awx/main/middleware.py b/awx/main/middleware.py index 90739aebbe..64e8110fc6 100644 --- a/awx/main/middleware.py +++ b/awx/main/middleware.py @@ -26,6 +26,17 @@ logger = logging.getLogger('awx.main.middleware') perf_logger = logging.getLogger('awx.analytics.performance') +class SettingsCacheMiddleware(MiddlewareMixin): + """ + Clears the in-memory settings cache at the beginning of a request. + We do this so that a script can POST to /api/v2/settings/all/ and then + right away GET /api/v2/settings/all/ and see the updated value. + """ + + def process_request(self, request): + settings._awx_conf_memoizedcache.clear() + + class TimingMiddleware(threading.local, MiddlewareMixin): dest = '/var/log/tower/profile' diff --git a/awx/main/tests/functional/commands/test_secret_key_regeneration.py b/awx/main/tests/functional/commands/test_secret_key_regeneration.py index 6bd0f0a576..f223f5d80d 100644 --- a/awx/main/tests/functional/commands/test_secret_key_regeneration.py +++ b/awx/main/tests/functional/commands/test_secret_key_regeneration.py @@ -52,10 +52,12 @@ class TestKeyRegeneration: settings.cache.delete('REDHAT_PASSWORD') # verify that the old SECRET_KEY doesn't work + settings._awx_conf_memoizedcache.clear() with pytest.raises(InvalidToken): settings.REDHAT_PASSWORD # verify that the new SECRET_KEY *does* work + settings._awx_conf_memoizedcache.clear() with override_settings(SECRET_KEY=new_key): assert settings.REDHAT_PASSWORD == 'sensitive' diff --git a/awx/main/tests/functional/test_notifications.py b/awx/main/tests/functional/test_notifications.py index 881241ffb2..ce3873c223 100644 --- a/awx/main/tests/functional/test_notifications.py +++ b/awx/main/tests/functional/test_notifications.py @@ -10,6 +10,8 @@ from awx.main.models.notifications import NotificationTemplate, Notification from awx.main.models.inventory import Inventory, InventorySource from awx.main.models.jobs import JobTemplate +from django.test.utils import override_settings + @pytest.mark.django_db def test_get_notification_template_list(get, user, notification_template): @@ -163,7 +165,7 @@ def test_custom_environment_injection(post, user, organization): ) assert response.status_code == 201 template = NotificationTemplate.objects.get(pk=response.data['id']) - with pytest.raises(ConnectionError), mock.patch('django.conf.settings.AWX_TASK_ENV', {'HTTPS_PROXY': '192.168.50.100:1234'}), mock.patch.object( + with pytest.raises(ConnectionError), override_settings(AWX_TASK_ENV={'HTTPS_PROXY': '192.168.50.100:1234'}), mock.patch.object( HTTPAdapter, 'send' ) as fake_send: diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 0f528ada14..24b4ca79ff 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -955,6 +955,7 @@ RECEPTOR_RELEASE_WORK = True MIDDLEWARE = [ 'django_guid.middleware.guid_middleware', + 'awx.main.middleware.SettingsCacheMiddleware', 'awx.main.middleware.TimingMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'awx.main.middleware.MigrationRanCheckMiddleware',