add an awx-manage command for re-generating SECRET_KEY

awx/main/management/commands/regenerate_secret_key.py

@@ -0,0 +1,129 @@
+import base64
+import json
+import os
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.db import transaction
+from django.db.models.signals import post_save
+
+from awx.conf import settings_registry
+from awx.conf.models import Setting
+from awx.conf.signals import on_post_save_setting
+from awx.main.models import (
+    UnifiedJob, Credential, NotificationTemplate, Job, JobTemplate, WorkflowJob,
+    WorkflowJobTemplate, OAuth2Application
+)
+from awx.main.utils.encryption import (
+    encrypt_field, decrypt_field, encrypt_value, decrypt_value, get_encryption_key
+)
+
+
+class Command(BaseCommand):
+    """
+    Regenerate a new SECRET_KEY value and re-encrypt every secret in the
+    Tower database.
+    """
+
+    @transaction.atomic
+    def handle(self, **options):
+        self.old_key = settings.SECRET_KEY
+        self.new_key = base64.encodebytes(os.urandom(33)).decode().rstrip()
+        self._notification_templates()
+        self._credentials()
+        self._unified_jobs()
+        self._oauth2_app_secrets()
+        self._settings()
+        self._survey_passwords()
+        return self.new_key
+
+    def _notification_templates(self):
+        for nt in NotificationTemplate.objects.iterator():
+            CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NotificationTemplate.NOTIFICATION_TYPES])
+            notification_class = CLASS_FOR_NOTIFICATION_TYPE[nt.notification_type]
+            for field in filter(lambda x: notification_class.init_parameters[x]['type'] == "password",
+                                notification_class.init_parameters):
+                nt.notification_configuration[field] = decrypt_field(nt, 'notification_configuration', subfield=field, secret_key=self.old_key)
+                nt.notification_configuration[field] = encrypt_field(nt, 'notification_configuration', subfield=field, secret_key=self.new_key)
+            nt.save()
+
+    def _credentials(self):
+        for credential in Credential.objects.iterator():
+            for field_name in credential.credential_type.secret_fields:
+                if field_name in credential.inputs:
+                    credential.inputs[field_name] = decrypt_field(
+                        credential,
+                        field_name,
+                        secret_key=self.old_key
+                    )
+                    credential.inputs[field_name] = encrypt_field(
+                        credential,
+                        field_name,
+                        secret_key=self.new_key
+                    )
+                credential.save()
+
+    def _unified_jobs(self):
+        for uj in UnifiedJob.objects.iterator():
+            if uj.start_args:
+                uj.start_args = decrypt_field(
+                    uj,
+                    'start_args',
+                    secret_key=self.old_key
+                )
+                uj.start_args = encrypt_field(uj, 'start_args', secret_key=self.new_key)
+                uj.save()
+
+    def _oauth2_app_secrets(self):
+        for app in OAuth2Application.objects.iterator():
+            raw = app.client_secret
+            app.client_secret = raw
+            encrypted = encrypt_value(raw, secret_key=self.new_key)
+            OAuth2Application.objects.filter(pk=app.pk).update(client_secret=encrypted)
+
+    def _settings(self):
+        # don't update memcached, the *actual* value isn't changing
+        post_save.disconnect(on_post_save_setting, sender=Setting)
+        for setting in Setting.objects.filter().order_by('pk'):
+            if settings_registry.is_setting_encrypted(setting.key):
+                setting.value = decrypt_field(setting, 'value', secret_key=self.old_key)
+                setting.value = encrypt_field(setting, 'value', secret_key=self.new_key)
+                setting.save()
+
+    def _survey_passwords(self):
+        for _type in (JobTemplate, WorkflowJobTemplate):
+            for jt in _type.objects.exclude(survey_spec={}):
+                changed = False
+                if jt.survey_spec.get('spec', []):
+                    for field in jt.survey_spec['spec']:
+                        if field.get('type') == 'password' and field.get('default', ''):
+                            raw = decrypt_value(
+                                get_encryption_key('value', None, secret_key=self.old_key),
+                                field['default']
+                            )
+                            field['default'] = encrypt_value(
+                                raw,
+                                pk=None,
+                                secret_key=self.new_key
+                            )
+                            changed = True
+                if changed:
+                    jt.save(update_fields=["survey_spec"])
+
+        for _type in (Job, WorkflowJob):
+            for job in _type.objects.exclude(survey_passwords={}).iterator():
+                changed = False
+                for key in job.survey_passwords:
+                    if key in job.extra_vars:
+                        extra_vars = json.loads(job.extra_vars)
+                        if not extra_vars.get(key):
+                            continue
+                        raw = decrypt_value(
+                            get_encryption_key('value', None, secret_key=self.old_key),
+                            extra_vars[key]
+                        )
+                        extra_vars[key] = encrypt_value(raw, pk=None, secret_key=self.new_key)
+                        job.extra_vars = json.dumps(extra_vars)
+                        changed = True
+                if changed:
+                    job.save(update_fields=["extra_vars"])

awx/main/tests/functional/commands/test_secret_key_regeneration.py

@@ -0,0 +1,173 @@
+import json
+
+from cryptography.fernet import InvalidToken
+from django.test.utils import override_settings
+from django.conf import settings
+import pytest
+
+from awx.main import models
+from awx.conf.models import Setting
+from awx.main.management.commands import regenerate_secret_key
+from awx.main.utils.encryption import encrypt_field, decrypt_field, encrypt_value
+
+
+PREFIX = '$encrypted$UTF8$AESCBC$'
+
+
+@pytest.mark.django_db
+class TestKeyRegeneration:
+
+    def test_encrypted_ssh_password(self, credential):
+        # test basic decryption
+        assert credential.inputs['password'].startswith(PREFIX)
+        assert credential.get_input('password') == 'secret'
+
+        # re-key the credential
+        new_key = regenerate_secret_key.Command().handle()
+        new_cred = models.Credential.objects.get(pk=credential.pk)
+        assert credential.inputs['password'] != new_cred.inputs['password']
+
+        # verify that the old SECRET_KEY doesn't work
+        with pytest.raises(InvalidToken):
+            new_cred.get_input('password')
+
+        # verify that the new SECRET_KEY *does* work
+        with override_settings(SECRET_KEY=new_key):
+            assert new_cred.get_input('password') == 'secret'
+
+    def test_encrypted_setting_values(self):
+        # test basic decryption
+        settings.LOG_AGGREGATOR_PASSWORD = 'sensitive'
+        s = Setting.objects.filter(key='LOG_AGGREGATOR_PASSWORD').first()
+        assert s.value.startswith(PREFIX)
+        assert settings.LOG_AGGREGATOR_PASSWORD == 'sensitive'
+
+        # re-key the setting value
+        new_key = regenerate_secret_key.Command().handle()
+        new_setting = Setting.objects.filter(key='LOG_AGGREGATOR_PASSWORD').first()
+        assert s.value != new_setting.value
+
+        # wipe out the local cache so the value is pulled from the DB again
+        settings.cache.delete('LOG_AGGREGATOR_PASSWORD')
+
+        # verify that the old SECRET_KEY doesn't work
+        with pytest.raises(InvalidToken):
+            settings.LOG_AGGREGATOR_PASSWORD
+
+        # verify that the new SECRET_KEY *does* work
+        with override_settings(SECRET_KEY=new_key):
+            assert settings.LOG_AGGREGATOR_PASSWORD == 'sensitive'
+
+    def test_encrypted_notification_secrets(self, notification_template_with_encrypt):
+        # test basic decryption
+        nt = notification_template_with_encrypt
+        nc = nt.notification_configuration
+        assert nc['token'].startswith(PREFIX)
+
+        Slack = nt.CLASS_FOR_NOTIFICATION_TYPE[nt.notification_type]
+        class TestBackend(Slack):
+
+            def __init__(self, *args, **kw):
+                assert kw['token'] == 'token'
+
+            def send_messages(self, messages):
+                pass
+
+        nt.CLASS_FOR_NOTIFICATION_TYPE['test'] = TestBackend
+        nt.notification_type = 'test'
+        nt.send('Subject', 'Body')
+
+        # re-key the notification config
+        new_key = regenerate_secret_key.Command().handle()
+        new_nt = models.NotificationTemplate.objects.get(pk=nt.pk)
+        assert nt.notification_configuration['token'] != new_nt.notification_configuration['token']
+
+        # verify that the old SECRET_KEY doesn't work
+        with pytest.raises(InvalidToken):
+            new_nt.CLASS_FOR_NOTIFICATION_TYPE['test'] = TestBackend
+            new_nt.notification_type = 'test'
+            new_nt.send('Subject', 'Body')
+
+        # verify that the new SECRET_KEY *does* work
+        with override_settings(SECRET_KEY=new_key):
+            new_nt.send('Subject', 'Body')
+
+    def test_job_start_args(self, job_factory):
+        # test basic decryption
+        job = job_factory()
+        job.start_args = json.dumps({'foo': 'bar'})
+        job.start_args = encrypt_field(job, field_name='start_args')
+        job.save()
+        assert job.start_args.startswith(PREFIX)
+
+        # re-key the start_args
+        new_key = regenerate_secret_key.Command().handle()
+        new_job = models.Job.objects.get(pk=job.pk)
+        assert new_job.start_args != job.start_args
+
+        # verify that the old SECRET_KEY doesn't work
+        with pytest.raises(InvalidToken):
+            decrypt_field(new_job, field_name='start_args')
+
+        # verify that the new SECRET_KEY *does* work
+        with override_settings(SECRET_KEY=new_key):
+            assert json.loads(
+                decrypt_field(new_job, field_name='start_args')
+            ) == {'foo': 'bar'}
+
+    @pytest.mark.parametrize('cls', ('JobTemplate', 'WorkflowJobTemplate'))
+    def test_survey_spec(self, inventory, project, survey_spec_factory, cls):
+        params = {}
+        if cls == 'JobTemplate':
+            params['inventory'] = inventory
+            params['project'] = project
+        # test basic decryption
+        jt = getattr(models, cls).objects.create(
+            name='Example Template',
+            survey_spec=survey_spec_factory([{
+                'variable': 'secret_key',
+                'default': encrypt_value('donttell', pk=None),
+                'type': 'password'
+            }]),
+            survey_enabled=True,
+            **params
+        )
+        job = jt.create_unified_job()
+        assert jt.survey_spec['spec'][0]['default'].startswith(PREFIX)
+        assert job.survey_passwords == {'secret_key': '$encrypted$'}
+        assert json.loads(job.decrypted_extra_vars())['secret_key'] == 'donttell'
+
+        # re-key the extra_vars
+        new_key = regenerate_secret_key.Command().handle()
+        new_job = models.UnifiedJob.objects.get(pk=job.pk)
+        assert new_job.extra_vars != job.extra_vars
+
+        # verify that the old SECRET_KEY doesn't work
+        with pytest.raises(InvalidToken):
+            new_job.decrypted_extra_vars()
+
+        # verify that the new SECRET_KEY *does* work
+        with override_settings(SECRET_KEY=new_key):
+            assert json.loads(
+                new_job.decrypted_extra_vars()
+            )['secret_key'] == 'donttell'
+
+    def test_oauth2_application_client_secret(self, oauth_application):
+        # test basic decryption
+        secret = oauth_application.client_secret
+        assert len(secret) == 128
+
+        # re-key the client_secret
+        new_key = regenerate_secret_key.Command().handle()
+
+        # verify that the old SECRET_KEY doesn't work
+        with pytest.raises(InvalidToken):
+            models.OAuth2Application.objects.get(
+                pk=oauth_application.pk
+            ).client_secret
+
+        # verify that the new SECRET_KEY *does* work
+        with override_settings(SECRET_KEY=new_key):
+            assert models.OAuth2Application.objects.get(
+                pk=oauth_application.pk
+            ).client_secret == secret

awx/main/utils/encryption.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import base64
 import hashlib
 import logging
@@ -35,7 +37,7 @@ class Fernet256(Fernet):
         self._backend = backend
 
 
-def get_encryption_key(field_name, pk=None):
+def get_encryption_key(field_name, pk=None, secret_key=None):
     '''
     Generate key for encrypted password based on field name,
     ``settings.SECRET_KEY``, and instance pk (if available).
@@ -46,19 +48,57 @@ def get_encryption_key(field_name, pk=None):
     '''
     from django.conf import settings
     h = hashlib.sha512()
-    h.update(smart_bytes(settings.SECRET_KEY))
+    h.update(smart_bytes(secret_key or settings.SECRET_KEY))
     if pk is not None:
         h.update(smart_bytes(str(pk)))
     h.update(smart_bytes(field_name))
     return base64.urlsafe_b64encode(h.digest())
 
 
-def encrypt_value(value, pk=None):
+def encrypt_value(value, pk=None, secret_key=None):
+    #
+    # ⚠️  D-D-D-DANGER ZONE ⚠️
+    #
+    # !!! BEFORE USING THIS FUNCTION PLEASE READ encrypt_field !!!
+    #
     TransientField = namedtuple('TransientField', ['pk', 'value'])
-    return encrypt_field(TransientField(pk=pk, value=value), 'value')
+    return encrypt_field(TransientField(pk=pk, value=value), 'value', secret_key=secret_key)
 
 
-def encrypt_field(instance, field_name, ask=False, subfield=None):
+def encrypt_field(instance, field_name, ask=False, subfield=None, secret_key=None):
+    #
+    # ⚠️  D-D-D-DANGER ZONE ⚠️
+    #
+    # !!! PLEASE READ BEFORE USING THIS FUNCTION ANYWHERE !!!
+    #
+    # You should know that this function is used in various places throughout
+    # AWX for symmetric encryption - generally it's used to encrypt sensitive
+    # values that we store in the AWX database (such as SSH private keys for
+    # credentials).
+    #
+    # If you're reading this function's code because you're thinking about
+    # using it to encrypt *something new*, please remember that AWX has
+    # official support for *regenerating* the SECRET_KEY (on which the
+    # symmetric key is based):
+    #
+    # $ awx-manage regenerate_secret_key
+    #
+    # ...so you'll need to *also* add code to support the
+    # migration/re-encryption of these values (the code in question lives in
+    # `awx.main.management.commands.regenerate_secret_key`):
+    #
+    # For example, if you find that you're adding a new database column that is
+    # encrypted, in addition to calling `encrypt_field` in the appropriate
+    # places, you would also need to update the `awx-manage regenerate_secret_key`
+    # so that values are properly migrated when the SECRET_KEY changes.
+    #
+    # This process *generally* involves adding Python code to the
+    # `regenerate_secret_key` command, i.e.,
+    #
+    # 1.  Query the database for existing encrypted values on the appropriate object(s)
+    # 2.  Decrypting them using the *old* SECRET_KEY
+    # 3.  Storing newly encrypted values using the *newly generated* SECRET_KEY
+    #
     '''
     Return content of the given instance and field name encrypted.
     '''
@@ -76,7 +116,11 @@ def encrypt_field(instance, field_name, ask=False, subfield=None):
     value = smart_str(value)
     if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'):
         return value
-    key = get_encryption_key(field_name, getattr(instance, 'pk', None))
+    key = get_encryption_key(
+        field_name,
+        getattr(instance, 'pk', None),
+        secret_key=secret_key
+    )
     f = Fernet256(key)
     encrypted = f.encrypt(smart_bytes(value))
     b64data = smart_str(base64.b64encode(encrypted))
@@ -99,7 +143,7 @@ def decrypt_value(encryption_key, value):
     return smart_str(value)
 
 
-def decrypt_field(instance, field_name, subfield=None):
+def decrypt_field(instance, field_name, subfield=None, secret_key=None):
     '''
     Return content of the given instance and field name decrypted.
     '''
@@ -115,7 +159,11 @@ def decrypt_field(instance, field_name, subfield=None):
     value = smart_str(value)
     if not value or not value.startswith('$encrypted$'):
         return value
-    key = get_encryption_key(field_name, getattr(instance, 'pk', None))
+    key = get_encryption_key(
+        field_name,
+        getattr(instance, 'pk', None),
+        secret_key=secret_key
+    )
 
     try:
         return smart_str(decrypt_value(key, value))

installer/roles/kubernetes/tasks/main.yml

@@ -314,4 +314,4 @@
     {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
       scale {{ deployment_object }} {{ kubernetes_deployment_name }} --replicas=0
     {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
-      scale {{ deployment_object }} {{ kubernetes_deployment_name }} --replicas={{ kubernetes_deployment_replica_size }}
+      scale {{ deployment_object }} {{ kubernetes_deployment_name }} --replicas={{ replicas | default(kubernetes_deployment_replica_size) }}

installer/roles/kubernetes/tasks/rekey.yml

@@ -0,0 +1,72 @@
+---
+- include_tasks: openshift_auth.yml
+  when: openshift_host is defined
+
+- include_tasks: kubernetes_auth.yml
+  when: kubernetes_context is defined
+
+- name: Use kubectl or oc
+  set_fact:
+    kubectl_or_oc: "{{ openshift_oc_bin if openshift_oc_bin is defined else 'kubectl' }}"
+
+- set_fact:
+    deployment_object: "sts"
+
+- name: Record deployment size
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      get {{ deployment_object }} {{ kubernetes_deployment_name }} -o jsonpath="{.status.replicas}"
+  register: deployment_size
+
+- name: Scale deployment down
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      scale {{ deployment_object }} {{ kubernetes_deployment_name }} --replicas=0
+
+- name: Wait for scale down
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} get pods \
+      -o jsonpath='{.items[*].metadata.name}' \
+      | tr -s '[[:space:]]' '\n' \
+      | grep {{ kubernetes_deployment_name }} \
+      | grep -v postgres | wc -l
+  register: tower_pods
+  until: (tower_pods.stdout | trim) == '0'
+  retries: 30
+
+- name: Delete any existing management pod
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      delete pod ansible-tower-management --grace-period=0 --ignore-not-found
+
+- name: Template management pod
+  set_fact:
+    management_pod: "{{ lookup('template', 'management-pod.yml.j2') }}"
+
+- name: Create management pod
+  shell: |
+    echo {{ management_pod | quote }} | {{ kubectl_or_oc }} apply -f -
+
+- name: Wait for management pod to start
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      get pod ansible-tower-management -o jsonpath="{.status.phase}"
+  register: result
+  until: result.stdout == "Running"
+  retries: 60
+  delay: 10
+
+- name: generate a new SECRET_KEY
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      exec -i ansible-tower-management -- bash -c "awx-manage regenerate_secret_key"
+  register: new_key
+
+- name: print the new SECRET_KEY
+  debug:
+    msg: "{{ new_key.stdout }}"
+
+- name: Delete management pod
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      delete pod ansible-tower-management --grace-period=0 --ignore-not-found