From 2c7cb4a370bb847b43b98fd09ef358cd73af6daa Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 10 Feb 2017 14:30:42 -0500 Subject: [PATCH 1/3] add utf-8 support to utils.common.encrypt_field/decrypt_field --- .../tests/unit/utils/common/test_common.py | 19 ++++++++++++--- awx/main/utils/common.py | 23 ++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/awx/main/tests/unit/utils/common/test_common.py b/awx/main/tests/unit/utils/common/test_common.py index a152d69c12..a48bbe64b3 100644 --- a/awx/main/tests/unit/utils/common/test_common.py +++ b/awx/main/tests/unit/utils/common/test_common.py @@ -1,24 +1,37 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Ansible, Inc. +# All Rights Reserved. + from awx.conf.models import Setting from awx.main.utils import common def test_encrypt_field(): field = Setting(pk=123, value='ANSIBLE') - encrypted = common.encrypt_field(field, 'value') + encrypted = field.value = common.encrypt_field(field, 'value') assert encrypted == '$encrypted$AES$Ey83gcmMuBBT1OEq2lepnw==' assert common.decrypt_field(field, 'value') == 'ANSIBLE' def test_encrypt_field_without_pk(): field = Setting(value='ANSIBLE') - encrypted = common.encrypt_field(field, 'value') + encrypted = field.value = common.encrypt_field(field, 'value') assert encrypted == '$encrypted$AES$8uIzEoGyY6QJwoTWbMFGhw==' assert common.decrypt_field(field, 'value') == 'ANSIBLE' +def test_encrypt_field_with_unicode_string(): + value = u'Iñtërnâtiônàlizætiøn' + field = Setting(value=value) + encrypted = field.value = common.encrypt_field(field, 'value') + assert encrypted == '$encrypted$UTF8$AES$AESQbqOefpYcLC7x8yZ2aWG4FlXlS66JgavLbDp/DSM=' + assert common.decrypt_field(field, 'value') == value + + def test_encrypt_subfield(): field = Setting(value={'name': 'ANSIBLE'}) - encrypted = common.encrypt_field(field, 'value', subfield='name') + encrypted = field.value = common.encrypt_field(field, 'value', subfield='name') assert encrypted == '$encrypted$AES$8uIzEoGyY6QJwoTWbMFGhw==' assert common.decrypt_field(field, 'value', subfield='name') == 'ANSIBLE' diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 5842b78db9..e49f4d0131 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -21,6 +21,8 @@ import tempfile # Decorator from decorator import decorator +import six + # Django from django.utils.translation import ugettext_lazy as _ from django.db.models import ManyToManyField @@ -190,6 +192,7 @@ def encrypt_field(instance, field_name, ask=False, subfield=None): value = value[subfield] if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'): return value + utf8 = type(value) == six.text_type value = smart_str(value) key = get_encryption_key(field_name, getattr(instance, 'pk', None)) cipher = AES.new(key, AES.MODE_ECB) @@ -197,17 +200,31 @@ def encrypt_field(instance, field_name, ask=False, subfield=None): value += '\x00' encrypted = cipher.encrypt(value) b64data = base64.b64encode(encrypted) - return '$encrypted$%s$%s' % ('AES', b64data) + tokens = ['$encrypted', 'AES', b64data] + if utf8: + # If the value to encrypt is utf-8, we need to add a marker so we + # know to decode the data when it's decrypted later + tokens.insert(1, 'UTF8') + return '$'.join(tokens) def decrypt_value(encryption_key, value): - algo, b64data = value[len('$encrypted$'):].split('$', 1) + raw_data = value[len('$encrypted$'):] + # If the encrypted string contains a UTF8 marker, discard it + utf8 = raw_data.startswith('UTF8$') + if utf8: + raw_data = raw_data[len('UTF8$'):] + algo, b64data = raw_data.split('$', 1) if algo != 'AES': raise ValueError('unsupported algorithm: %s' % algo) encrypted = base64.b64decode(b64data) cipher = AES.new(encryption_key, AES.MODE_ECB) value = cipher.decrypt(encrypted) - return value.rstrip('\x00') + value = value.rstrip('\x00') + # If the encrypted string contained a UTF8 marker, decode the data + if utf8: + value = value.decode('utf-8') + return value def decrypt_field(instance, field_name, subfield=None): From 64a973ae0249f7995c6800d932499c4710eb0295 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 10 Feb 2017 10:41:28 -0500 Subject: [PATCH 2/3] work around a unicode handling bug in python-memcached that affects py2 see: https://github.com/linsomniac/python-memcached/issues/79 see: #5276 --- awx/conf/settings.py | 14 ++++++++++- awx/conf/tests/unit/test_settings.py | 37 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/awx/conf/settings.py b/awx/conf/settings.py index 8b1c0786a1..43ad9ee61f 100644 --- a/awx/conf/settings.py +++ b/awx/conf/settings.py @@ -6,6 +6,8 @@ import sys import threading import time +import six + # Django from django.conf import settings, UserSettingsHolder from django.core.cache import cache as django_cache @@ -88,7 +90,17 @@ class EncryptedCacheProxy(object): def get(self, key, **kwargs): value = self.cache.get(key, **kwargs) - return self._handle_encryption(self.decrypter, key, value) + value = self._handle_encryption(self.decrypter, key, value) + + # python-memcached auto-encodes unicode on cache set in python2 + # https://github.com/linsomniac/python-memcached/issues/79 + # https://github.com/linsomniac/python-memcached/blob/288c159720eebcdf667727a859ef341f1e908308/memcache.py#L961 + if six.PY2 and isinstance(value, six.binary_type): + try: + six.text_type(value) + except UnicodeDecodeError: + value = value.decode('utf-8') + return value def set(self, key, value, **kwargs): self.cache.set( diff --git a/awx/conf/tests/unit/test_settings.py b/awx/conf/tests/unit/test_settings.py index 3fc78c452d..f7f1540108 100644 --- a/awx/conf/tests/unit/test_settings.py +++ b/awx/conf/tests/unit/test_settings.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # Copyright (c) 2017 Ansible, Inc. # All Rights Reserved. @@ -10,6 +12,7 @@ from django.core.cache.backends.locmem import LocMemCache from django.core.exceptions import ImproperlyConfigured from django.utils.translation import ugettext_lazy as _ import pytest +import six from awx.conf import models, fields from awx.conf.settings import SettingsWrapper, EncryptedCacheProxy, SETTING_CACHE_NOTSET @@ -60,6 +63,15 @@ def test_unregistered_setting(settings): assert settings.cache.get('DEBUG') is None +def test_cached_settings_unicode_is_auto_decoded(settings): + # https://github.com/linsomniac/python-memcached/issues/79 + # https://github.com/linsomniac/python-memcached/blob/288c159720eebcdf667727a859ef341f1e908308/memcache.py#L961 + + value = six.u('Iñtërnâtiônàlizætiøn').encode('utf-8') # this simulates what python-memcached does on cache.set() + settings.cache.set('DEBUG', value) + assert settings.cache.get('DEBUG') == six.u('Iñtërnâtiônàlizætiøn') + + def test_read_only_setting(settings): settings.registry.register( 'AWX_READ_ONLY', @@ -239,6 +251,31 @@ def test_setting_from_db(settings, mocker): assert settings.cache.get('AWX_SOME_SETTING') == 'FROM_DB' +@pytest.mark.parametrize('encrypted', (True, False)) +def test_setting_from_db_with_unicode(settings, mocker, encrypted): + settings.registry.register( + 'AWX_SOME_SETTING', + field_class=fields.CharField, + category=_('System'), + category_slug='system', + default='DEFAULT', + encrypted=encrypted + ) + # this simulates a bug in python-memcached; see https://github.com/linsomniac/python-memcached/issues/79 + value = six.u('Iñtërnâtiônàlizætiøn').encode('utf-8') + + setting_from_db = mocker.Mock(key='AWX_SOME_SETTING', value=value) + mocks = mocker.Mock(**{ + 'order_by.return_value': mocker.Mock(**{ + '__iter__': lambda self: iter([setting_from_db]), + 'first.return_value': setting_from_db + }), + }) + with mocker.patch('awx.conf.models.Setting.objects.filter', return_value=mocks): + assert settings.AWX_SOME_SETTING == six.u('Iñtërnâtiônàlizætiøn') + assert settings.cache.get('AWX_SOME_SETTING') == six.u('Iñtërnâtiônàlizætiøn') + + @pytest.mark.defined_in_file(AWX_SOME_SETTING='DEFAULT') def test_read_only_setting_assignment(settings): "read-only settings cannot be overwritten" From 5a8a647cf0376ec09565c1d411329900b8645e2c Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 13 Feb 2017 16:07:07 -0500 Subject: [PATCH 3/3] default log aggregator username and password to an empty string other configuration options seem to follow this pattern; the UI code seems to expect that it can send across an empty string see: #5276 --- awx/main/conf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/awx/main/conf.py b/awx/main/conf.py index d82e766607..6bb4c15895 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -258,7 +258,8 @@ register( register( 'LOG_AGGREGATOR_USERNAME', field_class=fields.CharField, - allow_null=True, + allow_blank=True, + default='', label=_('Logging Aggregator Username'), help_text=_('Username for external log aggregator (if required).'), category=_('Logging'), @@ -268,7 +269,8 @@ register( register( 'LOG_AGGREGATOR_PASSWORD', field_class=fields.CharField, - allow_null=True, + allow_blank=True, + default='', encrypted=True, label=_('Logging Aggregator Password/Token'), help_text=_('Password or authentication token for external log aggregator (if required).'),