From 659d31324db4a0323e09f14fb8618924141f7f7e Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 11 Dec 2017 16:07:13 -0500 Subject: [PATCH] hide survey passwords in saved launch configs --- awx/api/serializers.py | 96 ++++++++++++------- .../migrations/0012_non_blank_workflow.py | 23 +++++ awx/main/models/jobs.py | 11 ++- awx/main/models/mixins.py | 4 +- awx/main/models/unified_jobs.py | 4 +- awx/main/models/workflow.py | 3 - .../functional/api/test_job_runtime_params.py | 4 +- .../serializers/test_workflow_serializers.py | 44 +++++++++ 8 files changed, 142 insertions(+), 47 deletions(-) create mode 100644 awx/main/migrations/0012_non_blank_workflow.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 2a8423b508..83dddd6cef 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -46,6 +46,7 @@ from awx.main.utils import ( camelcase_to_underscore, getattrd, parse_yaml_or_json, has_model_field_prefetched, extract_ansible_vars) from awx.main.utils.filters import SmartFilter +from awx.main.redact import REPLACE_STR from awx.main.validators import vars_validate_or_raise @@ -3028,6 +3029,7 @@ class LaunchConfigurationBaseSerializer(BaseSerializer): diff_mode = serializers.NullBooleanField(required=False, default=None) verbosity = serializers.ChoiceField(allow_null=True, required=False, default=None, choices=VERBOSITY_CHOICES) + exclude_errors = () class Meta: fields = ('*', 'extra_data', 'inventory', # Saved launch-time config fields @@ -3055,10 +3057,47 @@ class LaunchConfigurationBaseSerializer(BaseSerializer): attrs.pop(field_name) return mock_obj + def to_representation(self, obj): + ret = super(LaunchConfigurationBaseSerializer, self).to_representation(obj) + if obj is None: + return ret + if 'extra_data' in ret and obj.survey_passwords: + ret['extra_data'] = obj.display_extra_data() + return ret + def validate(self, attrs): attrs = super(LaunchConfigurationBaseSerializer, self).validate(attrs) - # Verify that fields do not violate template's prompting rules - attrs['char_prompts'] = self._build_mock_obj(attrs).char_prompts + + # Build unsaved version of this config, use it to detect prompts errors + ujt = None + if 'unified_job_template' in attrs: + ujt = attrs['unified_job_template'] + elif self.instance: + ujt = self.instance.unified_job_template + mock_obj = self._build_mock_obj(attrs) + accepted, rejected, errors = ujt._accept_or_ignore_job_kwargs( + _exclude_errors=self.exclude_errors, **mock_obj.prompts_dict()) + + # Launch configs call extra_vars extra_data for historical reasons + if 'extra_vars' in errors: + errors['extra_data'] = errors.pop('extra_vars') + if errors: + raise serializers.ValidationError(errors) + + # Model `.save` needs the container dict, not the psuedo fields + attrs['char_prompts'] = mock_obj.char_prompts + + # Insert survey_passwords to track redacted variables + # TODO: perform encryption on save + if 'extra_data' in attrs: + extra_data = parse_yaml_or_json(attrs.get('extra_data', {})) + if hasattr(ujt, 'survey_password_variables'): + password_dict = {} + for key in ujt.survey_password_variables(): + if key in extra_data: + password_dict[key] = REPLACE_STR + if not self.instance or password_dict != self.instance.survey_passwords: + attrs['survey_passwords'] = password_dict return attrs @@ -3069,6 +3108,7 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer): success_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True) failure_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True) always_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True) + exclude_errors = ('required') # required variables may be provided by WFJT or on launch class Meta: model = WorkflowJobTemplateNode @@ -3082,8 +3122,10 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer): res['always_nodes'] = self.reverse('api:workflow_job_template_node_always_nodes_list', kwargs={'pk': obj.pk}) if obj.unified_job_template: res['unified_job_template'] = obj.unified_job_template.get_absolute_url(self.context.get('request')) - if obj.workflow_job_template: + try: res['workflow_job_template'] = self.reverse('api:workflow_job_template_detail', kwargs={'pk': obj.workflow_job_template.pk}) + except WorkflowJobTemplate.DoesNotExist: + pass return res def build_field(self, field_name, info, model_class, nested_depth): @@ -3094,32 +3136,28 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer): self.credential) return super(WorkflowJobTemplateNodeSerializer, self).build_field(field_name, info, model_class, nested_depth) + def build_relational_field(self, field_name, relation_info): + field_class, field_kwargs = super(WorkflowJobTemplateNodeSerializer, self).build_relational_field(field_name, relation_info) + # workflow_job_template is read-only unless creating a new node. + if self.instance and field_name == 'workflow_job_template': + field_kwargs['read_only'] = True + field_kwargs.pop('queryset', None) + return field_class, field_kwargs + def validate(self, attrs): deprecated_fields = {} if 'credential' in attrs: # TODO: remove when v2 API is deprecated deprecated_fields['credential'] = attrs.pop('credential') view = self.context.get('view') - if self.instance is None and ('workflow_job_template' not in attrs or - attrs['workflow_job_template'] is None): - raise serializers.ValidationError({ - "workflow_job_template": _("Workflow job template is missing during creation.") - }) + attrs = super(WorkflowJobTemplateNodeSerializer, self).validate(attrs) + ujt_obj = None if 'unified_job_template' in attrs: ujt_obj = attrs['unified_job_template'] - ujt_obj = None - if self.instance: + elif self.instance: ujt_obj = self.instance.unified_job_template if isinstance(ujt_obj, (WorkflowJobTemplate)): raise serializers.ValidationError({ "unified_job_template": _("Cannot nest a %s inside a WorkflowJobTemplate") % ujt_obj.__class__.__name__}) - attrs = super(WorkflowJobTemplateNodeSerializer, self).validate(attrs) - if ujt_obj is None: - ujt_obj = attrs.get('unified_job_template') - accepted, rejected, errors = ujt_obj._accept_or_ignore_job_kwargs(**self._build_mock_obj(attrs).prompts_dict()) - # Do not raise survey validation errors - errors.pop('variables_needed_to_start', None) - if errors: - raise serializers.ValidationError(errors) if 'credential' in deprecated_fields: # TODO: remove when v2 API is deprecated cred = deprecated_fields['credential'] attrs['credential'] = cred @@ -3467,8 +3505,9 @@ class JobLaunchSerializer(BaseSerializer): def validate(self, attrs): template = self.context.get('template') - template._is_manual_launch = True # signal to make several error types non-blocking - accepted, rejected, errors = template._accept_or_ignore_job_kwargs(**attrs) + accepted, rejected, errors = template._accept_or_ignore_job_kwargs( + _exclude_errors=['prompts', 'required'], # make several error types non-blocking + **attrs) self._ignored_fields = rejected if template.inventory and template.inventory.pending_deletion is True: @@ -3544,7 +3583,9 @@ class WorkflowJobLaunchSerializer(BaseSerializer): def validate(self, attrs): obj = self.instance - accepted, rejected, errors = obj._accept_or_ignore_job_kwargs(**attrs) + accepted, rejected, errors = obj._accept_or_ignore_job_kwargs( + _exclude_errors=['required'], + **attrs) WFJT_extra_vars = obj.extra_vars attrs = super(WorkflowJobLaunchSerializer, self).validate(attrs) @@ -3694,19 +3735,6 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer): 'Schedule its source project `{}` instead.'.format(value.source_project.name))) return value - def validate(self, attrs): - ujt = None - if 'unified_job_template' in attrs: - ujt = attrs['unified_job_template'] - elif self.instance: - ujt = self.instance.unified_job_template - accepted, rejected, errors = ujt._accept_or_ignore_job_kwargs(**self._build_mock_obj(attrs).prompts_dict()) - if 'extra_vars' in errors: - errors['extra_data'] = errors.pop('extra_vars') - if errors: - raise serializers.ValidationError(errors) - return super(ScheduleSerializer, self).validate(attrs) - # We reject rrules if: # - DTSTART is not include # - INTERVAL is not included diff --git a/awx/main/migrations/0012_non_blank_workflow.py b/awx/main/migrations/0012_non_blank_workflow.py new file mode 100644 index 0000000000..b863b8460f --- /dev/null +++ b/awx/main/migrations/0012_non_blank_workflow.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2017-12-11 16:40 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0011_blank_start_args'), + ] + + operations = [ + migrations.AlterField( + model_name='workflowjobtemplatenode', + name='workflow_job_template', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='workflow_job_template_nodes', to='main.WorkflowJobTemplate'), + preserve_default=False, + ), + ] diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 9b3d198533..cecb2755b0 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -350,9 +350,12 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour not variables_needed) def _accept_or_ignore_job_kwargs(self, **kwargs): + exclude_errors = kwargs.pop('_exclude_errors', []) prompted_data = {} rejected_data = {} - accepted_vars, rejected_vars, errors_dict = self.accept_or_ignore_variables(kwargs.get('extra_vars', {})) + accepted_vars, rejected_vars, errors_dict = self.accept_or_ignore_variables( + kwargs.get('extra_vars', {}), + _exclude_errors=exclude_errors) if accepted_vars: prompted_data['extra_vars'] = accepted_vars if rejected_vars: @@ -389,10 +392,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour rejected_data[field_name] = new_value # Not considered an error for manual launch, to support old # behavior of putting them in ignored_fields and launching anyway - if not getattr(self, '_is_manual_launch', False): + if 'prompts' not in exclude_errors: errors_dict[field_name] = _('Field is not configured to prompt on launch.').format(field_name=field_name) - if not getattr(self, '_is_manual_launch', False) and self.passwords_needed_to_start: + if 'prompts' not in exclude_errors and self.passwords_needed_to_start: errors_dict['passwords_needed_to_start'] = _( 'Saved launch configurations cannot provide passwords needed to start.') @@ -1565,7 +1568,7 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions): rejected_data['extra_vars'] = rejected_vars return (prompted_data, rejected_data, errors) - def _accept_or_ignore_variables(self, data, errors): + def _accept_or_ignore_variables(self, data, errors, _exclude_errors=()): ''' Unlike other templates, like project updates and inventory sources, system job templates can accept a limited number of fields diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 95077ee699..5b1c327b12 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -233,7 +233,7 @@ class SurveyJobTemplateMixin(models.Model): choice_list)) return errors - def _accept_or_ignore_variables(self, data, errors=None): + def _accept_or_ignore_variables(self, data, errors=None, _exclude_errors=()): survey_is_enabled = (self.survey_enabled and self.survey_spec) extra_vars = data.copy() if errors is None: @@ -266,7 +266,7 @@ class SurveyJobTemplateMixin(models.Model): # Leftover extra_vars, keys provided that are not allowed rejected.update(extra_vars) # ignored variables does not block manual launch - if not getattr(self, '_is_manual_launch', False): + if 'prompts' not in _exclude_errors: errors['extra_vars'] = [_('Variables {list_of_keys} are not allowed on launch.').format( list_of_keys=', '.join(extra_vars.keys()))] diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 821089d98d..47df3f4edb 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -435,7 +435,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio errors[field_name] = [_("Field is not allowed on launch.")] return ({}, kwargs, errors) - def accept_or_ignore_variables(self, data, errors=None): + def accept_or_ignore_variables(self, data, errors=None, _exclude_errors=()): ''' If subclasses accept any `variables` or `extra_vars`, they should define _accept_or_ignore_variables to place those variables in the accepted dict, @@ -453,7 +453,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio # SurveyJobTemplateMixin cannot override any methods because of # resolution order, forced by how metaclass processes fields, # thus the need for hasattr check - return self._accept_or_ignore_variables(data, errors) + return self._accept_or_ignore_variables(data, errors, _exclude_errors=_exclude_errors) elif data: errors['extra_vars'] = [ _('Variables {list_of_keys} provided, but this template cannot accept variables.'.format( diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 021a9672c6..78978f998c 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -113,9 +113,6 @@ class WorkflowJobTemplateNode(WorkflowNodeBase): workflow_job_template = models.ForeignKey( 'WorkflowJobTemplate', related_name='workflow_job_template_nodes', - blank=True, - null=True, - default=None, on_delete=models.CASCADE, ) diff --git a/awx/main/tests/functional/api/test_job_runtime_params.py b/awx/main/tests/functional/api/test_job_runtime_params.py index 3170ce01e0..402547b1a6 100644 --- a/awx/main/tests/functional/api/test_job_runtime_params.py +++ b/awx/main/tests/functional/api/test_job_runtime_params.py @@ -488,8 +488,8 @@ def test_job_launch_JT_with_credentials(machine_credential, credential, net_cred assert validated, serializer.errors kv['credentials'] = [credential, net_credential, machine_credential] # convert to internal value - prompted_fields, ignored_fields, errors = deploy_jobtemplate._accept_or_ignore_job_kwargs(**kv) - deploy_jobtemplate._is_manual_launch = True + prompted_fields, ignored_fields, errors = deploy_jobtemplate._accept_or_ignore_job_kwargs( + _exclude_errors=['required', 'prompts'], **kv) job_obj = deploy_jobtemplate.create_unified_job(**prompted_fields) creds = job_obj.credentials.all() diff --git a/awx/main/tests/unit/api/serializers/test_workflow_serializers.py b/awx/main/tests/unit/api/serializers/test_workflow_serializers.py index b87aa277b4..7e19f66420 100644 --- a/awx/main/tests/unit/api/serializers/test_workflow_serializers.py +++ b/awx/main/tests/unit/api/serializers/test_workflow_serializers.py @@ -13,6 +13,10 @@ from awx.main.models import ( WorkflowJobTemplateNode, WorkflowJob, WorkflowJobNode, + WorkflowJobTemplate, + Project, + Inventory, + JobTemplate ) @@ -150,6 +154,46 @@ class TestWorkflowJobTemplateNodeSerializerCharPrompts(): assert WFJT_serializer.instance.limit == 'webservers' +@mock.patch('awx.api.serializers.BaseSerializer.validate', lambda self, attrs: attrs) +class TestWorkflowJobTemplateNodeSerializerSurveyPasswords(): + + @pytest.fixture + def jt(self, survey_spec_factory): + return JobTemplate( + name='fake-jt', + survey_enabled=True, + survey_spec=survey_spec_factory(variables='var1', default_type='password'), + project=Project('fake-proj'), project_id=42, + inventory=Inventory('fake-inv'), inventory_id=42 + ) + + def test_set_survey_passwords_create(self, jt): + serializer = WorkflowJobTemplateNodeSerializer() + wfjt = WorkflowJobTemplate(name='fake-wfjt') + attrs = serializer.validate({ + 'unified_job_template': jt, + 'workflow_job_template': wfjt, + 'extra_data': {'var1': 'secret_answer'} + }) + assert 'survey_passwords' in attrs + assert 'var1' in attrs['survey_passwords'] + + def test_set_survey_passwords_modify(self, jt): + serializer = WorkflowJobTemplateNodeSerializer() + wfjt = WorkflowJobTemplate(name='fake-wfjt') + serializer.instance = WorkflowJobTemplateNode( + workflow_job_template=wfjt, + unified_job_template=jt + ) + attrs = serializer.validate({ + 'unified_job_template': jt, + 'workflow_job_template': wfjt, + 'extra_data': {'var1': 'secret_answer'} + }) + assert 'survey_passwords' in attrs + assert 'var1' in attrs['survey_passwords'] + + @mock.patch('awx.api.serializers.WorkflowJobTemplateNodeSerializer.get_related', lambda x,y: {}) class TestWorkflowJobNodeSerializerGetRelated(): @pytest.fixture