work around a unicode handling bug in python-memcached that affects py2

see: https://github.com/linsomniac/python-memcached/issues/79
see: #5276

awx/conf/settings.py

@@ -6,6 +6,8 @@ import sys
 import threading
 import time
 
+import six
+
 # Django
 from django.conf import settings, UserSettingsHolder
 from django.core.cache import cache as django_cache
@@ -88,7 +90,17 @@ class EncryptedCacheProxy(object):
 
     def get(self, key, **kwargs):
         value = self.cache.get(key, **kwargs)
-        return self._handle_encryption(self.decrypter, key, value)
+        value = self._handle_encryption(self.decrypter, key, value)
+
+        # python-memcached auto-encodes unicode on cache set in python2
+        # https://github.com/linsomniac/python-memcached/issues/79
+        # https://github.com/linsomniac/python-memcached/blob/288c159720eebcdf667727a859ef341f1e908308/memcache.py#L961
+        if six.PY2 and isinstance(value, six.binary_type):
+            try:
+                six.text_type(value)
+            except UnicodeDecodeError:
+                value = value.decode('utf-8')
+        return value
 
     def set(self, key, value, **kwargs):
         self.cache.set(

awx/conf/tests/unit/test_settings.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
@@ -10,6 +12,7 @@ from django.core.cache.backends.locmem import LocMemCache
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.translation import ugettext_lazy as _
 import pytest
+import six
 
 from awx.conf import models, fields
 from awx.conf.settings import SettingsWrapper, EncryptedCacheProxy, SETTING_CACHE_NOTSET
@@ -60,6 +63,15 @@ def test_unregistered_setting(settings):
     assert settings.cache.get('DEBUG') is None
 
 
+def test_cached_settings_unicode_is_auto_decoded(settings):
+    # https://github.com/linsomniac/python-memcached/issues/79
+    # https://github.com/linsomniac/python-memcached/blob/288c159720eebcdf667727a859ef341f1e908308/memcache.py#L961
+
+    value = six.u('Iñtërnâtiônàlizætiøn').encode('utf-8')  # this simulates what python-memcached does on cache.set()
+    settings.cache.set('DEBUG', value)
+    assert settings.cache.get('DEBUG') == six.u('Iñtërnâtiônàlizætiøn')
+
+
 def test_read_only_setting(settings):
     settings.registry.register(
         'AWX_READ_ONLY',
@@ -239,6 +251,31 @@ def test_setting_from_db(settings, mocker):
         assert settings.cache.get('AWX_SOME_SETTING') == 'FROM_DB'
 
 
+@pytest.mark.parametrize('encrypted', (True, False))
+def test_setting_from_db_with_unicode(settings, mocker, encrypted):
+    settings.registry.register(
+        'AWX_SOME_SETTING',
+        field_class=fields.CharField,
+        category=_('System'),
+        category_slug='system',
+        default='DEFAULT',
+        encrypted=encrypted
+    )
+    # this simulates a bug in python-memcached; see https://github.com/linsomniac/python-memcached/issues/79
+    value = six.u('Iñtërnâtiônàlizætiøn').encode('utf-8')
+
+    setting_from_db = mocker.Mock(key='AWX_SOME_SETTING', value=value)
+    mocks = mocker.Mock(**{
+        'order_by.return_value': mocker.Mock(**{
+            '__iter__': lambda self: iter([setting_from_db]),
+            'first.return_value': setting_from_db
+        }),
+    })
+    with mocker.patch('awx.conf.models.Setting.objects.filter', return_value=mocks):
+        assert settings.AWX_SOME_SETTING == six.u('Iñtërnâtiônàlizætiøn')
+        assert settings.cache.get('AWX_SOME_SETTING') == six.u('Iñtërnâtiônàlizætiøn')
+
+
 @pytest.mark.defined_in_file(AWX_SOME_SETTING='DEFAULT')
 def test_read_only_setting_assignment(settings):
     "read-only settings cannot be overwritten"