Relocate encryption helpers, update settings, tests, and imports

awx/conf/settings.py

@@ -52,7 +52,7 @@ SETTING_CACHE_TIMEOUT = 60
 # Flag indicating whether to store field default values in the cache.
 SETTING_CACHE_DEFAULTS = True
 
-__all__ = ['SettingsWrapper']
+__all__ = ['SettingsWrapper', 'get_settings_to_cache', 'SETTING_CACHE_NOTSET']
 
 
 @contextlib.contextmanager
@@ -147,6 +147,27 @@ class EncryptedCacheProxy(object):
         setattr(self.cache, name, value)
 
 
+def get_writeable_settings(registry):
+    return registry.get_registered_settings(read_only=False)
+
+
+def get_settings_to_cache(registry):
+    return dict([(key, SETTING_CACHE_NOTSET) for key in get_writeable_settings(registry)])
+
+
+def get_cache_value(value):
+    '''Returns the proper special cache setting for a value
+    based on instance type.
+    '''
+    if value is None:
+        value = SETTING_CACHE_NONE
+    elif isinstance(value, (list, tuple)) and len(value) == 0:
+        value = SETTING_CACHE_EMPTY_LIST
+    elif isinstance(value, (dict,)) and len(value) == 0:
+        value = SETTING_CACHE_EMPTY_DICT
+    return value
+
+
 class SettingsWrapper(UserSettingsHolder):
 
     @classmethod
@@ -188,18 +209,6 @@ class SettingsWrapper(UserSettingsHolder):
     def _get_supported_settings(self):
         return self.registry.get_registered_settings()
 
-    def _get_writeable_settings(self):
-        return self.registry.get_registered_settings(read_only=False)
-
-    def _get_cache_value(self, value):
-        if value is None:
-            value = SETTING_CACHE_NONE
-        elif isinstance(value, (list, tuple)) and len(value) == 0:
-            value = SETTING_CACHE_EMPTY_LIST
-        elif isinstance(value, (dict,)) and len(value) == 0:
-            value = SETTING_CACHE_EMPTY_DICT
-        return value
-
     def _preload_cache(self):
         # Ensure we're only modifying local preload timeout from one thread.
         with self._awx_conf_preload_lock:
@@ -212,7 +221,7 @@ class SettingsWrapper(UserSettingsHolder):
             # make those read-only to avoid overriding in the database.
             if not self._awx_conf_init_readonly and 'migrate_to_database_settings' not in sys.argv:
                 defaults_snapshot = self._get_default('DEFAULTS_SNAPSHOT')
-                for key in self._get_writeable_settings():
+                for key in get_writeable_settings(self.registry):
                     init_default = defaults_snapshot.get(key, None)
                     try:
                         file_default = self._get_default(key)
@@ -230,7 +239,7 @@ class SettingsWrapper(UserSettingsHolder):
         # Initialize all database-configurable settings with a marker value so
         # to indicate from the cache that the setting is not configured without
         # a database lookup.
-        settings_to_cache = dict([(key, SETTING_CACHE_NOTSET) for key in self._get_writeable_settings()])
+        settings_to_cache = get_settings_to_cache(self.registry)
         # Load all settings defined in the database.
         for setting in Setting.objects.filter(key__in=settings_to_cache.keys(), user__isnull=True).order_by('pk'):
             if settings_to_cache[setting.key] != SETTING_CACHE_NOTSET:
@@ -239,7 +248,7 @@ class SettingsWrapper(UserSettingsHolder):
                 value = decrypt_field(setting, 'value')
             else:
                 value = setting.value
-            settings_to_cache[setting.key] = self._get_cache_value(value)
+            settings_to_cache[setting.key] = get_cache_value(value)
         # Load field default value for any settings not found in the database.
         if SETTING_CACHE_DEFAULTS:
             for key, value in settings_to_cache.items():
@@ -247,7 +256,7 @@ class SettingsWrapper(UserSettingsHolder):
                     continue
                 field = self.registry.get_setting_field(key)
                 try:
-                    settings_to_cache[key] = self._get_cache_value(field.get_default())
+                    settings_to_cache[key] = get_cache_value(field.get_default())
                 except SkipField:
                     pass
         # Generate a cache key for each setting and store them all at once.
@@ -296,9 +305,9 @@ class SettingsWrapper(UserSettingsHolder):
                 value = SETTING_CACHE_NOTSET
             if cache_value != value:
                 logger.debug('cache set(%r, %r, %r)', cache_key,
-                             self._get_cache_value(value),
+                             get_cache_value(value),
                              SETTING_CACHE_TIMEOUT)
-                self.cache.set(cache_key, self._get_cache_value(value), timeout=SETTING_CACHE_TIMEOUT)
+                self.cache.set(cache_key, get_cache_value(value), timeout=SETTING_CACHE_TIMEOUT)
         if value == SETTING_CACHE_NOTSET and not SETTING_CACHE_DEFAULTS:
             try:
                 value = field.get_default()

awx/conf/tests/__init__.py

@@ -1,2 +0,0 @@
-# Copyright (c) 2017 Ansible, Inc.
-# All Rights Reserved.

awx/main/fields.py

@@ -470,7 +470,7 @@ class CredentialInputField(JSONSchemaField):
                 v != '$encrypted$',
                 model_instance.pk
             ]):
-                decrypted_values[k] = utils.common.decrypt_field(model_instance, k)
+                decrypted_values[k] = utils.decrypt_field(model_instance, k)
             else:
                 decrypted_values[k] = v
 

awx/main/migrations/_credentialtypes.py

@@ -1,6 +1,6 @@
 from awx.main import utils
 from awx.main.models import CredentialType
-from awx.main.utils.common import encrypt_field, decrypt_field
+from awx.main.utils import encrypt_field, decrypt_field
 from django.db.models import Q
 
 

awx/main/tests/functional/api/test_credential.py

@@ -2,7 +2,7 @@ import mock # noqa
 import pytest
 
 from awx.main.models.credential import Credential, CredentialType
-from awx.main.utils.common import decrypt_field
+from awx.main.utils import decrypt_field
 from awx.api.versioning import reverse
 
 EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY-----'

awx/main/tests/functional/test_credential.py

@@ -4,7 +4,7 @@
 import pytest
 from django.core.exceptions import ValidationError
 
-from awx.main.utils.common import decrypt_field
+from awx.main.utils import decrypt_field
 from awx.main.models import Credential, CredentialType
 
 from rest_framework import serializers

awx/main/tests/functional/test_credential_migration.py

@@ -7,7 +7,7 @@ from django.apps import apps
 
 from awx.main.models import Credential, CredentialType
 from awx.main.migrations._credentialtypes import migrate_to_v2_credentials
-from awx.main.utils.common import decrypt_field
+from awx.main.utils import decrypt_field
 from awx.main.migrations._credentialtypes import _disassociate_non_insights_projects
 
 EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY-----'
@@ -319,7 +319,7 @@ def test_insights_migration():
             'username': 'bob',
             'password': 'some-password',
         })
-    
+
     assert cred.credential_type.name == 'Insights Basic Auth'
     assert cred.inputs['username'] == 'bob'
     assert cred.inputs['password'].startswith('$encrypted$')

awx/main/tests/unit/test_tasks.py

@@ -26,7 +26,7 @@ from awx.main.models import (
 
 from awx.main import tasks
 from awx.main.task_engine import TaskEnhancer
-from awx.main.utils.common import encrypt_field
+from awx.main.utils import encrypt_field
 
 
 @contextmanager

awx/main/tests/unit/utils/common/test_common.py

@@ -1,65 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (c) 2017 Ansible, Inc.
-# All Rights Reserved.
-import pytest
-
-from awx.conf.models import Setting
-from awx.main.utils import common
-
-
-def test_encrypt_field():
-    field = Setting(pk=123, value='ANSIBLE')
-    encrypted = field.value = common.encrypt_field(field, 'value')
-    assert encrypted == '$encrypted$AES$Ey83gcmMuBBT1OEq2lepnw=='
-    assert common.decrypt_field(field, 'value') == 'ANSIBLE'
-
-
-def test_encrypt_field_without_pk():
-    field = Setting(value='ANSIBLE')
-    encrypted = field.value = common.encrypt_field(field, 'value')
-    assert encrypted == '$encrypted$AES$8uIzEoGyY6QJwoTWbMFGhw=='
-    assert common.decrypt_field(field, 'value') == 'ANSIBLE'
-
-
-def test_encrypt_field_with_unicode_string():
-    value = u'Iñtërnâtiônàlizætiøn'
-    field = Setting(value=value)
-    encrypted = field.value = common.encrypt_field(field, 'value')
-    assert encrypted == '$encrypted$UTF8$AES$AESQbqOefpYcLC7x8yZ2aWG4FlXlS66JgavLbDp/DSM='
-    assert common.decrypt_field(field, 'value') == value
-
-
-def test_encrypt_field_force_disable_unicode():
-    value = u"NothingSpecial"
-    field = Setting(value=value)
-    encrypted = field.value = common.encrypt_field(field, 'value', skip_utf8=True)
-    assert "UTF8" not in encrypted
-    assert common.decrypt_field(field, 'value') == value
-
-
-def test_encrypt_subfield():
-    field = Setting(value={'name': 'ANSIBLE'})
-    encrypted = field.value = common.encrypt_field(field, 'value', subfield='name')
-    assert encrypted == '$encrypted$AES$8uIzEoGyY6QJwoTWbMFGhw=='
-    assert common.decrypt_field(field, 'value', subfield='name') == 'ANSIBLE'
-
-
-def test_encrypt_field_with_ask():
-    encrypted = common.encrypt_field(Setting(value='ASK'), 'value', ask=True)
-    assert encrypted == 'ASK'
-
-
-def test_encrypt_field_with_empty_value():
-    encrypted = common.encrypt_field(Setting(value=None), 'value')
-    assert encrypted is None
-
-
-@pytest.mark.parametrize('input_, output', [
-    ({"foo": "bar"}, {"foo": "bar"}),
-    ('{"foo": "bar"}', {"foo": "bar"}),
-    ('---\nfoo: bar', {"foo": "bar"}),
-    (4399, {}),
-])
-def test_parse_yaml_or_json(input_, output):
-    assert common.parse_yaml_or_json(input_) == output

awx/main/tests/unit/utils/test_common.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+import pytest
+
+from awx.main.utils import common
+
+
+@pytest.mark.parametrize('input_, output', [
+    ({"foo": "bar"}, {"foo": "bar"}),
+    ('{"foo": "bar"}', {"foo": "bar"}),
+    ('---\nfoo: bar', {"foo": "bar"}),
+    (4399, {}),
+])
+def test_parse_yaml_or_json(input_, output):
+    assert common.parse_yaml_or_json(input_) == output

awx/main/tests/unit/utils/test_encryption.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+from awx.conf.models import Setting
+from awx.main.utils import encryption
+
+
+def test_encrypt_field():
+    field = Setting(pk=123, value='ANSIBLE')
+    encrypted = field.value = encryption.encrypt_field(field, 'value')
+    assert encryption.decrypt_field(field, 'value') == 'ANSIBLE'
+    assert encrypted.startswith('$encrypted$AESCBC$')
+
+
+def test_encrypt_field_without_pk():
+    field = Setting(value='ANSIBLE')
+    encrypted = field.value = encryption.encrypt_field(field, 'value')
+    assert encryption.decrypt_field(field, 'value') == 'ANSIBLE'
+    assert encrypted.startswith('$encrypted$AESCBC$')
+
+
+def test_encrypt_field_with_unicode_string():
+    value = u'Iñtërnâtiônàlizætiøn'
+    field = Setting(value=value)
+    encrypted = field.value = encryption.encrypt_field(field, 'value')
+    assert encryption.decrypt_field(field, 'value') == value
+    assert encrypted.startswith('$encrypted$UTF8$AESCBC$')
+
+
+def test_encrypt_field_force_disable_unicode():
+    value = u"NothingSpecial"
+    field = Setting(value=value)
+    encrypted = field.value = encryption.encrypt_field(field, 'value', skip_utf8=True)
+    assert "UTF8" not in encrypted
+    assert encryption.decrypt_field(field, 'value') == value
+
+
+def test_encrypt_subfield():
+    field = Setting(value={'name': 'ANSIBLE'})
+    encrypted = field.value = encryption.encrypt_field(field, 'value', subfield='name')
+    assert encryption.decrypt_field(field, 'value', subfield='name') == 'ANSIBLE'
+    assert encrypted.startswith('$encrypted$AESCBC$')
+
+
+def test_encrypt_field_with_ask():
+    encrypted = encryption.encrypt_field(Setting(value='ASK'), 'value', ask=True)
+    assert encrypted == 'ASK'
+
+
+def test_encrypt_field_with_empty_value():
+    encrypted = encryption.encrypt_field(Setting(value=None), 'value')
+    assert encrypted is None

awx/main/utils/__init__.py

@@ -3,22 +3,4 @@
 
 # AWX
 from awx.main.utils.common import * # noqa
-
-# Fields that didn't get included in __all__
-# TODO: after initial commit of file move to devel, these can be added
-# to common.py __all__ and removed here
-from awx.main.utils.common import ( # noqa
-    RequireDebugTrueOrTest,
-    encrypt_field,
-    parse_yaml_or_json,
-    decrypt_field,
-    timestamp_apiformat,
-    model_instance_diff,
-    model_to_dict,
-    check_proot_installed,
-    build_proot_temp_dir,
-    wrap_args_with_proot,
-    get_system_task_capacity,
-    decrypt_field_value,
-    has_model_field_prefetched
-)
+from awx.main.utils.encryption import * # noqa

awx/main/utils/common.py

@@ -3,7 +3,6 @@
 
 # Python
 import base64
-import hashlib
 import json
 import yaml
 import logging
@@ -21,8 +20,6 @@ import tempfile
 # Decorator
 from decorator import decorator
 
-import six
-
 # Django
 from django.utils.translation import ugettext_lazy as _
 from django.db.models.fields.related import ForeignObjectRel, ManyToManyField
@@ -33,9 +30,6 @@ from django.utils.encoding import smart_str
 from django.utils.text import slugify
 from django.apps import apps
 
-# PyCrypto
-from Crypto.Cipher import AES
-
 logger = logging.getLogger('awx.main.utils')
 
 __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore', 'memoize',
@@ -45,7 +39,10 @@ __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore',
            'ignore_inventory_computed_fields', 'ignore_inventory_group_removal',
            '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'NoDefaultProvided',
            'get_current_apps', 'set_current_apps', 'OutputEventFilter',
-           'callback_filter_out_ansible_extra_vars', 'get_search_fields',]
+           'callback_filter_out_ansible_extra_vars', 'get_search_fields', 'get_system_task_capacity',
+           'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict',
+           'model_instance_diff', 'timestamp_apiformat', 'parse_yaml_or_json', 'RequireDebugTrueOrTest',
+           'has_model_field_prefetched']
 
 
 def get_object_or_400(klass, *args, **kwargs):
@@ -164,90 +161,6 @@ def get_awx_version():
         return __version__
 
 
-def get_encryption_key(field_name, pk=None):
-    '''
-    Generate key for encrypted password based on field name,
-    ``settings.SECRET_KEY``, and instance pk (if available).
-
-    :param pk: (optional) the primary key of the ``awx.conf.model.Setting``;
-               can be omitted in situations where you're encrypting a setting
-               that is not database-persistent (like a read-only setting)
-    '''
-    from django.conf import settings
-    h = hashlib.sha1()
-    h.update(settings.SECRET_KEY)
-    if pk is not None:
-        h.update(str(pk))
-    h.update(field_name)
-    return h.digest()[:16]
-
-
-def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False):
-    '''
-    Return content of the given instance and field name encrypted.
-    '''
-    value = getattr(instance, field_name)
-    if isinstance(value, dict) and subfield is not None:
-        value = value[subfield]
-    if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'):
-        return value
-    if skip_utf8:
-        utf8 = False
-    else:
-        utf8 = type(value) == six.text_type
-    value = smart_str(value)
-    key = get_encryption_key(field_name, getattr(instance, 'pk', None))
-    cipher = AES.new(key, AES.MODE_ECB)
-    while len(value) % cipher.block_size != 0:
-        value += '\x00'
-    encrypted = cipher.encrypt(value)
-    b64data = base64.b64encode(encrypted)
-    tokens = ['$encrypted', 'AES', b64data]
-    if utf8:
-        # If the value to encrypt is utf-8, we need to add a marker so we
-        # know to decode the data when it's decrypted later
-        tokens.insert(1, 'UTF8')
-    return '$'.join(tokens)
-
-
-def decrypt_value(encryption_key, value):
-    raw_data = value[len('$encrypted$'):]
-    # If the encrypted string contains a UTF8 marker, discard it
-    utf8 = raw_data.startswith('UTF8$')
-    if utf8:
-        raw_data = raw_data[len('UTF8$'):]
-    algo, b64data = raw_data.split('$', 1)
-    if algo != 'AES':
-        raise ValueError('unsupported algorithm: %s' % algo)
-    encrypted = base64.b64decode(b64data)
-    cipher = AES.new(encryption_key, AES.MODE_ECB)
-    value = cipher.decrypt(encrypted)
-    value = value.rstrip('\x00')
-    # If the encrypted string contained a UTF8 marker, decode the data
-    if utf8:
-        value = value.decode('utf-8')
-    return value
-
-
-def decrypt_field(instance, field_name, subfield=None):
-    '''
-    Return content of the given instance and field name decrypted.
-    '''
-    value = getattr(instance, field_name)
-    if isinstance(value, dict) and subfield is not None:
-        value = value[subfield]
-    if not value or not value.startswith('$encrypted$'):
-        return value
-    key = get_encryption_key(field_name, getattr(instance, 'pk', None))
-
-    return decrypt_value(key, value)
-
-
-def decrypt_field_value(pk, field_name, value):
-    key = get_encryption_key(field_name, pk)
-    return decrypt_value(key, value)
-
-
 def update_scm_url(scm_type, url, username=True, password=True,
                    check_special_cases=True, scp_format=False):
     '''

awx/main/utils/encryption.py

@@ -0,0 +1,91 @@
+import base64
+import hashlib
+
+import six
+from cryptography.fernet import Fernet
+
+from django.utils.encoding import smart_str
+
+
+__all__ = ['get_encryption_key', 'encrypt_field', 'decrypt_field', 'decrypt_value', 'decrypt_field_value']
+
+
+def get_encryption_key(field_name, pk=None):
+    '''
+    Generate key for encrypted password based on field name,
+    ``settings.SECRET_KEY``, and instance pk (if available).
+
+    :param pk: (optional) the primary key of the ``awx.conf.model.Setting``;
+               can be omitted in situations where you're encrypting a setting
+               that is not database-persistent (like a read-only setting)
+    '''
+    from django.conf import settings
+    h = hashlib.sha256()
+    h.update(settings.SECRET_KEY)
+    if pk is not None:
+        h.update(str(pk))
+    h.update(field_name)
+    return base64.b64encode(h.digest())
+
+
+def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False):
+    '''
+    Return content of the given instance and field name encrypted.
+    '''
+    value = getattr(instance, field_name)
+    if isinstance(value, dict) and subfield is not None:
+        value = value[subfield]
+    if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'):
+        return value
+    if skip_utf8:
+        utf8 = False
+    else:
+        utf8 = type(value) == six.text_type
+    value = smart_str(value)
+    key = get_encryption_key(field_name, getattr(instance, 'pk', None))
+    f = Fernet(key)
+    encrypted = f.encrypt(value)
+    b64data = base64.b64encode(encrypted)
+    tokens = ['$encrypted', 'AESCBC', b64data]
+    if utf8:
+        # If the value to encrypt is utf-8, we need to add a marker so we
+        # know to decode the data when it's decrypted later
+        tokens.insert(1, 'UTF8')
+    return '$'.join(tokens)
+
+
+def decrypt_value(encryption_key, value):
+    raw_data = value[len('$encrypted$'):]
+    # If the encrypted string contains a UTF8 marker, discard it
+    utf8 = raw_data.startswith('UTF8$')
+    if utf8:
+        raw_data = raw_data[len('UTF8$'):]
+    algo, b64data = raw_data.split('$', 1)
+    if algo != 'AESCBC':
+        raise ValueError('unsupported algorithm: %s' % algo)
+    encrypted = base64.b64decode(b64data)
+    f = Fernet(encryption_key)
+    value = f.decrypt(encrypted)
+    # If the encrypted string contained a UTF8 marker, decode the data
+    if utf8:
+        value = value.decode('utf-8')
+    return value
+
+
+def decrypt_field(instance, field_name, subfield=None):
+    '''
+    Return content of the given instance and field name decrypted.
+    '''
+    value = getattr(instance, field_name)
+    if isinstance(value, dict) and subfield is not None:
+        value = value[subfield]
+    if not value or not value.startswith('$encrypted$'):
+        return value
+    key = get_encryption_key(field_name, getattr(instance, 'pk', None))
+
+    return decrypt_value(key, value)
+
+
+def decrypt_field_value(pk, field_name, value):
+    key = get_encryption_key(field_name, pk)
+    return decrypt_value(key, value)