Merge pull request #481 from ryanpetrello/fix-7046

[3.2.2] encrypt job survey data

awx/api/views.py

@@ -15,6 +15,7 @@ import logging
 import requests
 from base64 import b64encode
 from collections import OrderedDict
+import six
 
 # Django
 from django.conf import settings
@@ -72,6 +73,7 @@ from awx.main.utils import (
     extract_ansible_vars,
     decrypt_field,
 )
+from awx.main.utils.encryption import encrypt_value
 from awx.main.utils.filters import SmartFilter
 from awx.main.utils.insights import filter_insights_api_response
 
@@ -2899,8 +2901,15 @@ class JobTemplateSurveySpec(GenericAPIView):
             if "required" not in survey_item:
                 return Response(dict(error=_("'required' missing from survey question %s.") % str(idx)), status=status.HTTP_400_BAD_REQUEST)
 
-            if survey_item["type"] == "password":
-                if survey_item.get("default") and survey_item["default"].startswith('$encrypted$'):
+            if survey_item["type"] == "password" and "default" in survey_item:
+                if not isinstance(survey_item['default'], six.string_types):
+                    return Response(
+                        _("Value %s for '%s' expected to be a string." % (
+                            survey_item["default"], survey_item["variable"]
+                        )),
+                        status=status.HTTP_400_BAD_REQUEST
+                    )
+                elif survey_item["default"].startswith('$encrypted$'):
                     if not obj.survey_spec:
                         return Response(dict(error=_("$encrypted$ is reserved keyword and may not be used as a default for password {}.".format(str(idx)))),
                                         status=status.HTTP_400_BAD_REQUEST)
@@ -2909,6 +2918,8 @@ class JobTemplateSurveySpec(GenericAPIView):
                         for old_item in old_spec['spec']:
                             if old_item['variable'] == survey_item['variable']:
                                 survey_item['default'] = old_item['default']
+                else:
+                    survey_item['default'] = encrypt_value(survey_item['default'])
             idx += 1
 
         obj.survey_spec = new_spec

awx/main/migrations/0011_v322_encrypt_survey_passwords.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+from awx.main.migrations import ActivityStreamDisabledMigration
+from awx.main.migrations import _reencrypt as reencrypt
+
+
+class Migration(ActivityStreamDisabledMigration):
+
+    dependencies = [
+        ('main', '0010_v322_add_support_for_ovirt4_inventory'),
+    ]
+
+    operations = [
+        migrations.RunPython(reencrypt.encrypt_survey_passwords),
+    ]

awx/main/migrations/_reencrypt.py

@@ -1,5 +1,7 @@
 import logging
+import json
 from django.utils.translation import ugettext_lazy as _
+import six
 
 from awx.conf.migrations._reencrypt import (
     decrypt_field,
@@ -72,3 +74,38 @@ def _unified_jobs(apps):
                 uj.start_args = decrypt_field(uj, 'start_args')
                 uj.start_args = encrypt_field(uj, 'start_args')
                 uj.save()
+
+
+def encrypt_survey_passwords(apps, schema_editor):
+    _encrypt_survey_passwords(
+        apps.get_model('main', 'Job'),
+        apps.get_model('main', 'JobTemplate'),
+    )
+
+
+def _encrypt_survey_passwords(Job, JobTemplate):
+    from awx.main.utils.encryption import encrypt_value
+    for jt in JobTemplate.objects.exclude(survey_spec={}):
+        changed = False
+        if jt.survey_spec.get('spec', []):
+            for field in jt.survey_spec['spec']:
+                if field.get('type') == 'password' and field.get('default', ''):
+                    if field['default'].startswith('$encrypted$'):
+                        continue
+                    field['default'] = encrypt_value(field['default'], pk=None)
+                    changed = True
+        if changed:
+            jt.save()
+
+    for job in Job.objects.defer('result_stdout_text').exclude(survey_passwords={}).iterator():
+        changed = False
+        for key in job.survey_passwords:
+            if key in job.extra_vars:
+                extra_vars = json.loads(job.extra_vars)
+                if not extra_vars.get(key, '') or extra_vars[key].startswith('$encrypted$'):
+                    continue
+                extra_vars[key] = encrypt_value(extra_vars[key], pk=None)
+                job.extra_vars = json.dumps(extra_vars)
+                changed = True
+        if changed:
+            job.save()

awx/main/models/jobs.py

@@ -37,6 +37,7 @@ from awx.main.utils import (
     ignore_inventory_computed_fields,
     parse_yaml_or_json,
 )
+from awx.main.utils.encryption import encrypt_value
 from awx.main.fields import ImplicitRoleField
 from awx.main.models.mixins import ResourceMixin, SurveyJobTemplateMixin, SurveyJobMixin, TaskManagerJobMixin
 from awx.main.models.base import PERM_INVENTORY_SCAN
@@ -385,6 +386,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         # Sort the runtime fields allowed and disallowed by job template
         ignored_fields = {}
         prompted_fields = {}
+        survey_password_variables = self.survey_password_variables()
 
         ask_for_vars_dict = self._ask_for_vars_dict()
 
@@ -402,7 +404,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
                         extra_vars = parse_yaml_or_json(kwargs[field])
                         for key in extra_vars:
                             if key in survey_vars:
-                                prompted_fields[field][key] = extra_vars[key]
+                                if key in survey_password_variables:
+                                    prompted_fields[field][key] = encrypt_value(extra_vars[key])
+                                else:
+                                    prompted_fields[field][key] = extra_vars[key]
                             else:
                                 ignored_fields[field][key] = extra_vars[key]
                     else:

awx/main/models/mixins.py

@@ -13,6 +13,7 @@ from awx.main.models.rbac import (
     Role, RoleAncestorEntry, get_roles_on_resource
 )
 from awx.main.utils import parse_yaml_or_json
+from awx.main.utils.encryption import decrypt_value, get_encryption_key
 from awx.main.fields import JSONField
 
 
@@ -263,6 +264,20 @@ class SurveyJobMixin(models.Model):
         else:
             return self.extra_vars
 
+    def decrypted_extra_vars(self):
+        '''
+        Decrypts fields marked as passwords in survey.
+        '''
+        if self.survey_passwords:
+            extra_vars = json.loads(self.extra_vars)
+            for key in self.survey_passwords:
+                if key in extra_vars:
+                    value = extra_vars[key]
+                    extra_vars[key] = decrypt_value(get_encryption_key('value', pk=None), value)
+            return json.dumps(extra_vars)
+        else:
+            return self.extra_vars
+
 
 class TaskManagerUnifiedJobMixin(models.Model):
     class Meta:

awx/main/tasks.py

@@ -1180,7 +1180,7 @@ class RunJob(BaseTask):
             if kwargs.get('display', False) and job.job_template:
                 extra_vars.update(json.loads(job.display_extra_vars()))
             else:
-                extra_vars.update(job.extra_vars_dict)
+                extra_vars.update(json.loads(job.decrypted_extra_vars()))
         args.extend(['-e', json.dumps(extra_vars)])
 
         # Add path to playbook (relative to project.local_path).

awx/main/tests/functional/api/test_survey_spec.py

@@ -91,6 +91,111 @@ def test_survey_spec_sucessful_creation(survey_spec_factory, job_template, post,
     assert updated_jt.survey_spec == survey_input_data
 
 
+@mock.patch('awx.api.views.feature_enabled', lambda feature: True)
+@pytest.mark.django_db
+@pytest.mark.parametrize('value, status', [
+    ('SUPERSECRET', 201),
+    (['some', 'invalid', 'list'], 400),
+    ({'some-invalid': 'dict'}, 400),
+    (False, 400)
+])
+def test_survey_spec_passwords_are_encrypted_on_launch(job_template_factory, post, admin_user, value, status):
+    objects = job_template_factory('jt', organization='org1', project='prj',
+                                   inventory='inv', credential='cred')
+    job_template = objects.job_template
+    job_template.survey_enabled = True
+    job_template.save()
+    input_data = {
+        'description': 'A survey',
+        'spec': [{
+            'index': 0,
+            'question_name': 'What is your password?',
+            'required': True,
+            'variable': 'secret_value',
+            'type': 'password'
+        }],
+        'name': 'my survey'
+    }
+    post(url=reverse('api:job_template_survey_spec', kwargs={'pk': job_template.id}),
+         data=input_data, user=admin_user, expect=200)
+    resp = post(reverse('api:job_template_launch', kwargs={'pk': job_template.pk}),
+                dict(extra_vars=dict(secret_value=value)), admin_user, expect=status)
+
+    if status == 201:
+        job = Job.objects.get(pk=resp.data['id'])
+        assert json.loads(job.extra_vars)['secret_value'].startswith('$encrypted$')
+        assert json.loads(job.decrypted_extra_vars()) == {
+            'secret_value': value
+        }
+    else:
+        assert "for 'secret_value' expected to be a string." in json.dumps(resp.data)
+
+
+@mock.patch('awx.api.views.feature_enabled', lambda feature: True)
+@pytest.mark.django_db
+@pytest.mark.parametrize('default, status', [
+    ('SUPERSECRET', 200),
+    (['some', 'invalid', 'list'], 400),
+    ({'some-invalid': 'dict'}, 400),
+    (False, 400)
+])
+def test_survey_spec_default_passwords_are_encrypted(job_template, post, admin_user, default, status):
+    job_template.survey_enabled = True
+    job_template.save()
+    input_data = {
+        'description': 'A survey',
+        'spec': [{
+            'index': 0,
+            'question_name': 'What is your password?',
+            'required': True,
+            'variable': 'secret_value',
+            'default': default,
+            'type': 'password'
+        }],
+        'name': 'my survey'
+    }
+    resp = post(url=reverse('api:job_template_survey_spec', kwargs={'pk': job_template.id}),
+                data=input_data, user=admin_user, expect=status)
+
+    if status == 200:
+        updated_jt = JobTemplate.objects.get(pk=job_template.pk)
+        assert updated_jt.survey_spec['spec'][0]['default'].startswith('$encrypted$')
+
+        job = updated_jt.create_unified_job()
+        assert json.loads(job.extra_vars)['secret_value'].startswith('$encrypted$')
+        assert json.loads(job.decrypted_extra_vars()) == {
+            'secret_value': default
+        }
+    else:
+        assert "for 'secret_value' expected to be a string." in str(resp.data)
+
+
+@mock.patch('awx.api.views.feature_enabled', lambda feature: True)
+@pytest.mark.django_db
+def test_survey_spec_default_passwords_encrypted_on_update(job_template, post, put, admin_user):
+    input_data = {
+        'description': 'A survey',
+        'spec': [{
+            'index': 0,
+            'question_name': 'What is your password?',
+            'required': True,
+            'variable': 'secret_value',
+            'default': 'SUPERSECRET',
+            'type': 'password'
+        }],
+        'name': 'my survey'
+    }
+    post(url=reverse('api:job_template_survey_spec', kwargs={'pk': job_template.id}),
+         data=input_data, user=admin_user, expect=200)
+    updated_jt = JobTemplate.objects.get(pk=job_template.pk)
+
+    # simulate a survey field edit where we're not changing the default value
+    input_data['spec'][0]['default'] = '$encrypted$'
+    post(url=reverse('api:job_template_survey_spec', kwargs={'pk': job_template.id}),
+         data=input_data, user=admin_user, expect=200)
+    assert updated_jt.survey_spec == JobTemplate.objects.get(pk=job_template.pk).survey_spec
+
+
 # Tests related to survey content validation
 @mock.patch('awx.api.views.feature_enabled', lambda feature: True)
 @pytest.mark.django_db

awx/main/tests/functional/test_reencrypt_migration.py

@@ -10,6 +10,8 @@ from django.apps import apps
 
 from awx.main.models import (
     UnifiedJob,
+    Job,
+    JobTemplate,
     NotificationTemplate,
     Credential,
 )
@@ -20,9 +22,10 @@ from awx.main.migrations._reencrypt import (
     _notification_templates,
     _credentials,
     _unified_jobs,
+    _encrypt_survey_passwords
 )
 
-from awx.main.utils import decrypt_field
+from awx.main.utils import decrypt_field, get_encryption_key, decrypt_value
 
 
 @pytest.mark.django_db
@@ -93,3 +96,54 @@ def test_unified_job_migration(old_enc, new_enc, value):
     # Exception if the encryption type of AESCBC is not properly skipped, ensures
     # our `startswith` calls don't have typos
     _unified_jobs(apps)
+
+
+@pytest.mark.django_db
+def test_survey_default_password_encryption(job_template_factory):
+    jt = job_template_factory('jt', organization='org1', project='prj',
+                              inventory='inv', credential='cred').job_template
+    jt.survey_enabled = True
+    jt.survey_spec = {
+        'description': 'A survey',
+        'spec': [{
+            'index': 0,
+            'question_name': 'What is your password?',
+            'required': True,
+            'variable': 'secret_value',
+            'default': 'SUPERSECRET',
+            'type': 'password'
+        }],
+        'name': 'my survey'
+    }
+    jt.save()
+
+    _encrypt_survey_passwords(Job, JobTemplate)
+    spec = JobTemplate.objects.get(pk=jt.pk).survey_spec['spec']
+    assert decrypt_value(get_encryption_key('value', pk=None), spec[0]['default']) == 'SUPERSECRET'
+
+
+@pytest.mark.django_db
+def test_job_survey_vars_encryption(job_template_factory):
+    jt = job_template_factory('jt', organization='org1', project='prj',
+                              inventory='inv', credential='cred').job_template
+    jt.survey_enabled = True
+    jt.survey_spec = {
+        'description': 'A survey',
+        'spec': [{
+            'index': 0,
+            'question_name': 'What is your password?',
+            'required': True,
+            'variable': 'secret_value',
+            'default': '',
+            'type': 'password'
+        }],
+        'name': 'my survey'
+    }
+    jt.save()
+    job = jt.create_unified_job()
+    job.extra_vars = json.dumps({'secret_value': 'SUPERSECRET'})
+    job.save()
+
+    _encrypt_survey_passwords(Job, JobTemplate)
+    job = Job.objects.get(pk=job.pk)
+    assert json.loads(job.decrypted_extra_vars()) == {'secret_value': 'SUPERSECRET'}

awx/main/tests/unit/models/test_survey_models.py

@@ -13,6 +13,7 @@ from awx.main.models import (
 @pytest.fixture
 def job(mocker):
     ret = mocker.MagicMock(**{
+        'decrypted_extra_vars.return_value': '{\"secret_key\": \"my_password\"}',
         'display_extra_vars.return_value': '{\"secret_key\": \"$encrypted$\"}',
         'extra_vars_dict': {"secret_key": "my_password"},
         'pk': 1, 'job_template.pk': 1, 'job_template.name': '',

awx/main/tests/unit/test_tasks.py

@@ -31,7 +31,7 @@ from awx.main.models import (
 )
 
 from awx.main import tasks
-from awx.main.utils import encrypt_field
+from awx.main.utils import encrypt_field, encrypt_value
 
 
 
@@ -306,6 +306,20 @@ class TestGenericRun(TestJobExecution):
         assert '"awx_user_id": 123,' in ' '.join(args)
         assert '"awx_user_name": "angry-spud"' in ' '.join(args)
 
+    def test_survey_extra_vars(self):
+        self.instance.extra_vars = json.dumps({
+            'super_secret': encrypt_value('CLASSIFIED', pk=None)
+        })
+        self.instance.survey_passwords = {
+            'super_secret': '$encrypted$'
+        }
+        self.task.run(self.pk)
+
+        assert self.run_pexpect.call_count == 1
+        call_args, _ = self.run_pexpect.call_args_list[0]
+        args, cwd, env, stdout = call_args
+        assert '"super_secret": "CLASSIFIED"' in ' '.join(args)
+
     def test_awx_task_env(self):
         patch = mock.patch('awx.main.tasks.settings.AWX_TASK_ENV', {'FOO': 'BAR'})
         patch.start()

awx/main/utils/encryption.py

@@ -1,6 +1,7 @@
 import base64
 import hashlib
 import logging
+from collections import namedtuple
 
 import six
 from cryptography.fernet import Fernet, InvalidToken
@@ -8,7 +9,8 @@ from cryptography.hazmat.backends import default_backend
 from django.utils.encoding import smart_str
 
 
-__all__ = ['get_encryption_key', 'encrypt_field', 'decrypt_field', 'decrypt_value']
+__all__ = ['get_encryption_key', 'encrypt_value', 'encrypt_field',
+           'decrypt_field', 'decrypt_value']
 
 logger = logging.getLogger('awx.main.utils.encryption')
 
@@ -50,6 +52,11 @@ def get_encryption_key(field_name, pk=None):
     return base64.urlsafe_b64encode(h.digest())
 
 
+def encrypt_value(value, pk=None):
+    TransientField = namedtuple('TransientField', ['pk', 'value'])
+    return encrypt_field(TransientField(pk=pk, value=value), 'value')
+
+
 def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False):
     '''
     Return content of the given instance and field name encrypted.