Only use in-memory cache for database settings, set ttl=5 (#12166)

* Only use in-memory cache for database settings

Make necessary adjustments to monkeypatch
  as it is very vunerable to recursion
  Remove migration exception that is now redundant

Clear cache if a setting is changed

* Use dedicated middleware for setting cache stuff
  Clear cache for each request

* Add tests for in-memory cache

awx/__init__.py

@@ -78,9 +78,10 @@ def oauth2_getattribute(self, attr):
     # Custom method to override
     # oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__
     from django.conf import settings
+    from oauth2_provider.settings import DEFAULTS
 
     val = None
-    if 'migrate' not in sys.argv:
+    if (isinstance(attr, str)) and (attr in DEFAULTS) and (not attr.startswith('_')):
         # certain Django OAuth Toolkit migrations actually reference
         # setting lookups for references to model classes (e.g.,
         # oauth2_settings.REFRESH_TOKEN_MODEL)

awx/conf/settings.py

@@ -1,7 +1,6 @@
 # Python
 import contextlib
 import logging
-import sys
 import threading
 import time
 import os
@@ -31,7 +30,7 @@ from awx.conf.models import Setting
 
 logger = logging.getLogger('awx.conf.settings')
 
-SETTING_MEMORY_TTL = 5 if 'callback_receiver' in ' '.join(sys.argv) else 0
+SETTING_MEMORY_TTL = 5
 
 # Store a special value to indicate when a setting is not set in the database.
 SETTING_CACHE_NOTSET = '___notset___'
@@ -403,11 +402,15 @@ class SettingsWrapper(UserSettingsHolder):
         key=lambda *args, **kwargs: SettingsWrapper.hashkey(*args, **kwargs),
         lock=lambda self: self.__dict__['_awx_conf_memoizedcache_lock'],
     )
+    def _get_local_with_cache(self, name):
+        """Get value while accepting the in-memory cache if key is available"""
+        with _ctit_db_wrapper(trans_safe=True):
+            return self._get_local(name)
+
     def __getattr__(self, name):
         value = empty
         if name in self.all_supported_settings:
-            with _ctit_db_wrapper(trans_safe=True):
+            value = self._get_local_with_cache(name)
-                value = self._get_local(name)
         if value is not empty:
             return value
         return self._get_default(name)

awx/conf/signals.py

@@ -28,6 +28,9 @@ def handle_setting_change(key, for_delete=False):
     cache_keys = {Setting.get_cache_key(k) for k in setting_keys}
     cache.delete_many(cache_keys)
 
+    # if we have changed a setting, we want to avoid mucking with the in-memory cache entirely
+    settings._awx_conf_memoizedcache.clear()
+
     # Send setting_changed signal with new value for each setting.
     for setting_key in setting_keys:
         setting_changed.send(sender=Setting, setting=setting_key, value=getattr(settings, setting_key, None), enter=not bool(for_delete))

awx/conf/tests/unit/test_settings.py

@@ -8,6 +8,8 @@ import codecs
 from uuid import uuid4
 import time
 
+from unittest import mock
+
 from django.conf import LazySettings
 from django.core.cache.backends.locmem import LocMemCache
 from django.core.exceptions import ImproperlyConfigured
@@ -299,3 +301,33 @@ def test_readonly_sensitive_cache_data_is_encrypted(settings):
     cache.set('AWX_ENCRYPTED', 'SECRET!')
     assert cache.get('AWX_ENCRYPTED') == 'SECRET!'
     assert native_cache.get('AWX_ENCRYPTED') == 'FRPERG!'
+
+
+@pytest.mark.defined_in_file(AWX_VAR='DEFAULT')
+def test_in_memory_cache_only_for_registered_settings(settings):
+    "Test that we only make use of the in-memory TTL cache for registered settings"
+    settings._awx_conf_memoizedcache.clear()
+    settings.MIDDLEWARE
+    assert len(settings._awx_conf_memoizedcache) == 0  # does not cache MIDDLEWARE
+    settings.registry.register('AWX_VAR', field_class=fields.CharField, category=_('System'), category_slug='system')
+    settings._wrapped.__dict__['all_supported_settings'] = ['AWX_VAR']  # because it is cached_property
+    settings._awx_conf_memoizedcache.clear()
+    assert settings.AWX_VAR == 'DEFAULT'
+    assert len(settings._awx_conf_memoizedcache) == 1  # caches registered settings
+
+
+@pytest.mark.defined_in_file(AWX_VAR='DEFAULT')
+def test_in_memory_cache_works(settings):
+    settings._awx_conf_memoizedcache.clear()
+    settings.registry.register('AWX_VAR', field_class=fields.CharField, category=_('System'), category_slug='system')
+    settings._wrapped.__dict__['all_supported_settings'] = ['AWX_VAR']
+
+    settings._awx_conf_memoizedcache.clear()
+
+    with mock.patch('awx.conf.settings.SettingsWrapper._get_local', return_value='DEFAULT') as mock_get:
+        assert settings.AWX_VAR == 'DEFAULT'
+        mock_get.assert_called_once_with('AWX_VAR')
+
+    with mock.patch.object(settings, '_get_local') as mock_get:
+        assert settings.AWX_VAR == 'DEFAULT'
+        mock_get.assert_not_called()

awx/main/middleware.py

@@ -26,6 +26,17 @@ logger = logging.getLogger('awx.main.middleware')
 perf_logger = logging.getLogger('awx.analytics.performance')
 
 
+class SettingsCacheMiddleware(MiddlewareMixin):
+    """
+    Clears the in-memory settings cache at the beginning of a request.
+    We do this so that a script can POST to /api/v2/settings/all/ and then
+    right away GET /api/v2/settings/all/ and see the updated value.
+    """
+
+    def process_request(self, request):
+        settings._awx_conf_memoizedcache.clear()
+
+
 class TimingMiddleware(threading.local, MiddlewareMixin):
 
     dest = '/var/log/tower/profile'

awx/main/tests/functional/commands/test_secret_key_regeneration.py

@@ -52,10 +52,12 @@ class TestKeyRegeneration:
         settings.cache.delete('REDHAT_PASSWORD')
 
         # verify that the old SECRET_KEY doesn't work
+        settings._awx_conf_memoizedcache.clear()
         with pytest.raises(InvalidToken):
             settings.REDHAT_PASSWORD
 
         # verify that the new SECRET_KEY *does* work
+        settings._awx_conf_memoizedcache.clear()
         with override_settings(SECRET_KEY=new_key):
             assert settings.REDHAT_PASSWORD == 'sensitive'
 

awx/main/tests/functional/test_notifications.py

@@ -10,6 +10,8 @@ from awx.main.models.notifications import NotificationTemplate, Notification
 from awx.main.models.inventory import Inventory, InventorySource
 from awx.main.models.jobs import JobTemplate
 
+from django.test.utils import override_settings
+
 
 @pytest.mark.django_db
 def test_get_notification_template_list(get, user, notification_template):
@@ -163,7 +165,7 @@ def test_custom_environment_injection(post, user, organization):
     )
     assert response.status_code == 201
     template = NotificationTemplate.objects.get(pk=response.data['id'])
-    with pytest.raises(ConnectionError), mock.patch('django.conf.settings.AWX_TASK_ENV', {'HTTPS_PROXY': '192.168.50.100:1234'}), mock.patch.object(
+    with pytest.raises(ConnectionError), override_settings(AWX_TASK_ENV={'HTTPS_PROXY': '192.168.50.100:1234'}), mock.patch.object(
         HTTPAdapter, 'send'
     ) as fake_send:
 

awx/settings/defaults.py

@@ -955,6 +955,7 @@ RECEPTOR_RELEASE_WORK = True
 
 MIDDLEWARE = [
     'django_guid.middleware.guid_middleware',
+    'awx.main.middleware.SettingsCacheMiddleware',
     'awx.main.middleware.TimingMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'awx.main.middleware.MigrationRanCheckMiddleware',