hide survey passwords in saved launch configs

awx/api/serializers.py

@@ -46,6 +46,7 @@ from awx.main.utils import (
     camelcase_to_underscore, getattrd, parse_yaml_or_json,
     has_model_field_prefetched, extract_ansible_vars)
 from awx.main.utils.filters import SmartFilter
+from awx.main.redact import REPLACE_STR
 
 from awx.main.validators import vars_validate_or_raise
 
@@ -3028,6 +3029,7 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
     diff_mode = serializers.NullBooleanField(required=False, default=None)
     verbosity = serializers.ChoiceField(allow_null=True, required=False, default=None,
                                         choices=VERBOSITY_CHOICES)
+    exclude_errors = ()
 
     class Meta:
         fields = ('*', 'extra_data', 'inventory', # Saved launch-time config fields
@@ -3055,10 +3057,47 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
                 attrs.pop(field_name)
         return mock_obj
 
+    def to_representation(self, obj):
+        ret = super(LaunchConfigurationBaseSerializer, self).to_representation(obj)
+        if obj is None:
+            return ret
+        if 'extra_data' in ret and obj.survey_passwords:
+            ret['extra_data'] = obj.display_extra_data()
+        return ret
+
     def validate(self, attrs):
         attrs = super(LaunchConfigurationBaseSerializer, self).validate(attrs)
-        # Verify that fields do not violate template's prompting rules
-        attrs['char_prompts'] = self._build_mock_obj(attrs).char_prompts
+
+        # Build unsaved version of this config, use it to detect prompts errors
+        ujt = None
+        if 'unified_job_template' in attrs:
+            ujt = attrs['unified_job_template']
+        elif self.instance:
+            ujt = self.instance.unified_job_template
+        mock_obj = self._build_mock_obj(attrs)
+        accepted, rejected, errors = ujt._accept_or_ignore_job_kwargs(
+            _exclude_errors=self.exclude_errors, **mock_obj.prompts_dict())
+
+        # Launch configs call extra_vars extra_data for historical reasons
+        if 'extra_vars' in errors:
+            errors['extra_data'] = errors.pop('extra_vars')
+        if errors:
+            raise serializers.ValidationError(errors)
+
+        # Model `.save` needs the container dict, not the psuedo fields
+        attrs['char_prompts'] = mock_obj.char_prompts
+
+        # Insert survey_passwords to track redacted variables
+        # TODO: perform encryption on save
+        if 'extra_data' in attrs:
+            extra_data = parse_yaml_or_json(attrs.get('extra_data', {}))
+            if hasattr(ujt, 'survey_password_variables'):
+                password_dict = {}
+                for key in ujt.survey_password_variables():
+                    if key in extra_data:
+                        password_dict[key] = REPLACE_STR
+                if not self.instance or password_dict != self.instance.survey_passwords:
+                    attrs['survey_passwords'] = password_dict
         return attrs
 
 
@@ -3069,6 +3108,7 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer):
     success_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     failure_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     always_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
+    exclude_errors = ('required')  # required variables may be provided by WFJT or on launch
 
     class Meta:
         model = WorkflowJobTemplateNode
@@ -3082,8 +3122,10 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer):
         res['always_nodes'] = self.reverse('api:workflow_job_template_node_always_nodes_list', kwargs={'pk': obj.pk})
         if obj.unified_job_template:
             res['unified_job_template'] = obj.unified_job_template.get_absolute_url(self.context.get('request'))
-        if obj.workflow_job_template:
+        try:
             res['workflow_job_template'] = self.reverse('api:workflow_job_template_detail', kwargs={'pk': obj.workflow_job_template.pk})
+        except WorkflowJobTemplate.DoesNotExist:
+            pass
         return res
 
     def build_field(self, field_name, info, model_class, nested_depth):
@@ -3094,32 +3136,28 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer):
                                              self.credential)
         return super(WorkflowJobTemplateNodeSerializer, self).build_field(field_name, info, model_class, nested_depth)
 
+    def build_relational_field(self, field_name, relation_info):
+        field_class, field_kwargs = super(WorkflowJobTemplateNodeSerializer, self).build_relational_field(field_name, relation_info)
+        # workflow_job_template is read-only unless creating a new node.
+        if self.instance and field_name == 'workflow_job_template':
+            field_kwargs['read_only'] = True
+            field_kwargs.pop('queryset', None)
+        return field_class, field_kwargs
+
     def validate(self, attrs):
         deprecated_fields = {}
         if 'credential' in attrs:  # TODO: remove when v2 API is deprecated
             deprecated_fields['credential'] = attrs.pop('credential')
         view = self.context.get('view')
-        if self.instance is None and ('workflow_job_template' not in attrs or
-                                      attrs['workflow_job_template'] is None):
-            raise serializers.ValidationError({
-                "workflow_job_template": _("Workflow job template is missing during creation.")
-            })
+        attrs = super(WorkflowJobTemplateNodeSerializer, self).validate(attrs)
+        ujt_obj = None
         if 'unified_job_template' in attrs:
             ujt_obj = attrs['unified_job_template']
-        ujt_obj = None
-        if self.instance:
+        elif self.instance:
             ujt_obj = self.instance.unified_job_template
         if isinstance(ujt_obj, (WorkflowJobTemplate)):
             raise serializers.ValidationError({
                 "unified_job_template": _("Cannot nest a %s inside a WorkflowJobTemplate") % ujt_obj.__class__.__name__})
-        attrs = super(WorkflowJobTemplateNodeSerializer, self).validate(attrs)
-        if ujt_obj is None:
-            ujt_obj = attrs.get('unified_job_template')
-        accepted, rejected, errors = ujt_obj._accept_or_ignore_job_kwargs(**self._build_mock_obj(attrs).prompts_dict())
-        # Do not raise survey validation errors
-        errors.pop('variables_needed_to_start', None)
-        if errors:
-            raise serializers.ValidationError(errors)
         if 'credential' in deprecated_fields:  # TODO: remove when v2 API is deprecated
             cred = deprecated_fields['credential']
             attrs['credential'] = cred
@@ -3467,8 +3505,9 @@ class JobLaunchSerializer(BaseSerializer):
     def validate(self, attrs):
         template = self.context.get('template')
 
-        template._is_manual_launch = True  # signal to make several error types non-blocking
-        accepted, rejected, errors = template._accept_or_ignore_job_kwargs(**attrs)
+        accepted, rejected, errors = template._accept_or_ignore_job_kwargs(
+            _exclude_errors=['prompts', 'required'],  # make several error types non-blocking
+            **attrs)
         self._ignored_fields = rejected
 
         if template.inventory and template.inventory.pending_deletion is True:
@@ -3544,7 +3583,9 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
     def validate(self, attrs):
         obj = self.instance
 
-        accepted, rejected, errors = obj._accept_or_ignore_job_kwargs(**attrs)
+        accepted, rejected, errors = obj._accept_or_ignore_job_kwargs(
+            _exclude_errors=['required'],
+            **attrs)
 
         WFJT_extra_vars = obj.extra_vars
         attrs = super(WorkflowJobLaunchSerializer, self).validate(attrs)
@@ -3694,19 +3735,6 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer):
                 'Schedule its source project `{}` instead.'.format(value.source_project.name)))
         return value
 
-    def validate(self, attrs):
-        ujt = None
-        if 'unified_job_template' in attrs:
-            ujt = attrs['unified_job_template']
-        elif self.instance:
-            ujt = self.instance.unified_job_template
-        accepted, rejected, errors = ujt._accept_or_ignore_job_kwargs(**self._build_mock_obj(attrs).prompts_dict())
-        if 'extra_vars' in errors:
-            errors['extra_data'] = errors.pop('extra_vars')
-        if errors:
-            raise serializers.ValidationError(errors)
-        return super(ScheduleSerializer, self).validate(attrs)
-
     # We reject rrules if:
     # - DTSTART is not include
     # - INTERVAL is not included

awx/main/migrations/0012_non_blank_workflow.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2017-12-11 16:40
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0011_blank_start_args'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='workflowjobtemplatenode',
+            name='workflow_job_template',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='workflow_job_template_nodes', to='main.WorkflowJobTemplate'),
+            preserve_default=False,
+        ),
+    ]

awx/main/models/jobs.py

@@ -350,9 +350,12 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
                 not variables_needed)
 
     def _accept_or_ignore_job_kwargs(self, **kwargs):
+        exclude_errors = kwargs.pop('_exclude_errors', [])
         prompted_data = {}
         rejected_data = {}
-        accepted_vars, rejected_vars, errors_dict = self.accept_or_ignore_variables(kwargs.get('extra_vars', {}))
+        accepted_vars, rejected_vars, errors_dict = self.accept_or_ignore_variables(
+            kwargs.get('extra_vars', {}),
+            _exclude_errors=exclude_errors)
         if accepted_vars:
             prompted_data['extra_vars'] = accepted_vars
         if rejected_vars:
@@ -389,10 +392,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
                 rejected_data[field_name] = new_value
                 # Not considered an error for manual launch, to support old
                 # behavior of putting them in ignored_fields and launching anyway
-                if not getattr(self, '_is_manual_launch', False):
+                if 'prompts' not in exclude_errors:
                     errors_dict[field_name] = _('Field is not configured to prompt on launch.').format(field_name=field_name)
 
-        if not getattr(self, '_is_manual_launch', False) and self.passwords_needed_to_start:
+        if 'prompts' not in exclude_errors and self.passwords_needed_to_start:
             errors_dict['passwords_needed_to_start'] = _(
                 'Saved launch configurations cannot provide passwords needed to start.')
 
@@ -1565,7 +1568,7 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):
             rejected_data['extra_vars'] = rejected_vars
         return (prompted_data, rejected_data, errors)
 
-    def _accept_or_ignore_variables(self, data, errors):
+    def _accept_or_ignore_variables(self, data, errors, _exclude_errors=()):
         '''
         Unlike other templates, like project updates and inventory sources,
         system job templates can accept a limited number of fields

awx/main/models/mixins.py

@@ -233,7 +233,7 @@ class SurveyJobTemplateMixin(models.Model):
                                                                                    choice_list))
         return errors
 
-    def _accept_or_ignore_variables(self, data, errors=None):
+    def _accept_or_ignore_variables(self, data, errors=None, _exclude_errors=()):
         survey_is_enabled = (self.survey_enabled and self.survey_spec)
         extra_vars = data.copy()
         if errors is None:
@@ -266,7 +266,7 @@ class SurveyJobTemplateMixin(models.Model):
             # Leftover extra_vars, keys provided that are not allowed
             rejected.update(extra_vars)
             # ignored variables does not block manual launch
-            if not getattr(self, '_is_manual_launch', False):
+            if 'prompts' not in _exclude_errors:
                 errors['extra_vars'] = [_('Variables {list_of_keys} are not allowed on launch.').format(
                     list_of_keys=', '.join(extra_vars.keys()))]
 

awx/main/models/unified_jobs.py

@@ -435,7 +435,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio
                 errors[field_name] = [_("Field is not allowed on launch.")]
         return ({}, kwargs, errors)
 
-    def accept_or_ignore_variables(self, data, errors=None):
+    def accept_or_ignore_variables(self, data, errors=None, _exclude_errors=()):
         '''
         If subclasses accept any `variables` or `extra_vars`, they should
         define _accept_or_ignore_variables to place those variables in the accepted dict,
@@ -453,7 +453,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio
             # SurveyJobTemplateMixin cannot override any methods because of
             # resolution order, forced by how metaclass processes fields,
             # thus the need for hasattr check
-            return self._accept_or_ignore_variables(data, errors)
+            return self._accept_or_ignore_variables(data, errors, _exclude_errors=_exclude_errors)
         elif data:
             errors['extra_vars'] = [
                 _('Variables {list_of_keys} provided, but this template cannot accept variables.'.format(

awx/main/models/workflow.py

@@ -113,9 +113,6 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
     workflow_job_template = models.ForeignKey(
         'WorkflowJobTemplate',
         related_name='workflow_job_template_nodes',
-        blank=True,
-        null=True,
-        default=None,
         on_delete=models.CASCADE,
     )
 

awx/main/tests/functional/api/test_job_runtime_params.py

@@ -488,8 +488,8 @@ def test_job_launch_JT_with_credentials(machine_credential, credential, net_cred
     assert validated, serializer.errors
 
     kv['credentials'] = [credential, net_credential, machine_credential]  # convert to internal value
-    prompted_fields, ignored_fields, errors = deploy_jobtemplate._accept_or_ignore_job_kwargs(**kv)
-    deploy_jobtemplate._is_manual_launch = True
+    prompted_fields, ignored_fields, errors = deploy_jobtemplate._accept_or_ignore_job_kwargs(
+        _exclude_errors=['required', 'prompts'], **kv)
     job_obj = deploy_jobtemplate.create_unified_job(**prompted_fields)
 
     creds = job_obj.credentials.all()

awx/main/tests/unit/api/serializers/test_workflow_serializers.py

@@ -13,6 +13,10 @@ from awx.main.models import (
     WorkflowJobTemplateNode,
     WorkflowJob,
     WorkflowJobNode,
+    WorkflowJobTemplate,
+    Project,
+    Inventory,
+    JobTemplate
 )
 
 
@@ -150,6 +154,46 @@ class TestWorkflowJobTemplateNodeSerializerCharPrompts():
         assert WFJT_serializer.instance.limit == 'webservers'
 
 
+@mock.patch('awx.api.serializers.BaseSerializer.validate', lambda self, attrs: attrs)
+class TestWorkflowJobTemplateNodeSerializerSurveyPasswords():
+
+    @pytest.fixture
+    def jt(self, survey_spec_factory):
+        return JobTemplate(
+            name='fake-jt',
+            survey_enabled=True,
+            survey_spec=survey_spec_factory(variables='var1', default_type='password'),
+            project=Project('fake-proj'), project_id=42,
+            inventory=Inventory('fake-inv'), inventory_id=42
+        )
+
+    def test_set_survey_passwords_create(self, jt):
+        serializer = WorkflowJobTemplateNodeSerializer()
+        wfjt = WorkflowJobTemplate(name='fake-wfjt')
+        attrs = serializer.validate({
+            'unified_job_template': jt,
+            'workflow_job_template': wfjt,
+            'extra_data': {'var1': 'secret_answer'}
+        })
+        assert 'survey_passwords' in attrs
+        assert 'var1' in attrs['survey_passwords']
+
+    def test_set_survey_passwords_modify(self, jt):
+        serializer = WorkflowJobTemplateNodeSerializer()
+        wfjt = WorkflowJobTemplate(name='fake-wfjt')
+        serializer.instance = WorkflowJobTemplateNode(
+            workflow_job_template=wfjt,
+            unified_job_template=jt
+        )
+        attrs = serializer.validate({
+            'unified_job_template': jt,
+            'workflow_job_template': wfjt,
+            'extra_data': {'var1': 'secret_answer'}
+        })
+        assert 'survey_passwords' in attrs
+        assert 'var1' in attrs['survey_passwords']
+
+
 @mock.patch('awx.api.serializers.WorkflowJobTemplateNodeSerializer.get_related', lambda x,y: {})
 class TestWorkflowJobNodeSerializerGetRelated():
     @pytest.fixture