diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 1084ac8bbd..a353f67aca 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -87,8 +87,7 @@ SUMMARIZABLE_FK_FIELDS = { 'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), 'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed',), 'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud'), - 'cloud_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud'), - 'network_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'net'), + 'vault_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud'), 'job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'elapsed'), 'job_template': DEFAULT_SUMMARY_FIELDS, 'workflow_job_template': DEFAULT_SUMMARY_FIELDS, @@ -1843,6 +1842,7 @@ class ResourceAccessListElementSerializer(UserSerializer): class CredentialTypeSerializer(BaseSerializer): show_capabilities = ['edit', 'delete'] + managed_by_tower = serializers.ReadOnlyField() class Meta: model = CredentialType @@ -1850,6 +1850,9 @@ class CredentialTypeSerializer(BaseSerializer): 'injectors') def validate(self, attrs): + if self.instance and self.instance.managed_by_tower: + raise serializers.ValidationError( + {"detail": _("Modifications not allowed for credential types managed by Tower")}) fields = attrs.get('inputs', {}).get('fields', []) for field in fields: if field.get('ask_at_runtime', False): @@ -2105,14 +2108,42 @@ class LabelsListMixin(object): return res +# TODO: remove when API v1 is removed +@six.add_metaclass(BaseSerializerMetaclass) +class V1JobOptionsSerializer(BaseSerializer): + + class Meta: + model = Credential + fields = ('*', 'cloud_credential', 'network_credential') + + V1_FIELDS = { + 'cloud_credential': models.PositiveIntegerField(blank=True, null=True, default=None), + 'network_credential': models.PositiveIntegerField(blank=True, null=True, default=None) + } + + def build_field(self, field_name, info, model_class, nested_depth): + if field_name in self.V1_FIELDS: + return self.build_standard_field(field_name, + self.V1_FIELDS[field_name]) + return super(V1JobOptionsSerializer, self).build_field(field_name, info, model_class, nested_depth) + + class JobOptionsSerializer(LabelsListMixin, BaseSerializer): class Meta: fields = ('*', 'job_type', 'inventory', 'project', 'playbook', - 'credential', 'cloud_credential', 'network_credential', 'forks', 'limit', + 'credential', 'vault_credential', 'forks', 'limit', 'verbosity', 'extra_vars', 'job_tags', 'force_handlers', 'skip_tags', 'start_at_task', 'timeout', 'store_facts',) + def get_fields(self): + fields = super(JobOptionsSerializer, self).get_fields() + + # TODO: remove when API v1 is removed + if self.version == 1: + fields.update(V1JobOptionsSerializer().get_fields()) + return fields + def get_related(self, obj): res = super(JobOptionsSerializer, self).get_related(obj) res['labels'] = self.reverse('api:job_template_label_list', kwargs={'pk': obj.pk}) @@ -2122,12 +2153,19 @@ class JobOptionsSerializer(LabelsListMixin, BaseSerializer): res['project'] = self.reverse('api:project_detail', kwargs={'pk': obj.project.pk}) if obj.credential: res['credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.credential.pk}) - if obj.cloud_credential: - res['cloud_credential'] = self.reverse('api:credential_detail', - kwargs={'pk': obj.cloud_credential.pk}) - if obj.network_credential: - res['network_credential'] = self.reverse('api:credential_detail', - kwargs={'pk': obj.network_credential.pk}) + if obj.vault_credential: + res['vault_credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.vault_credential.pk}) + if self.version > 1: + view = 'api:%s_extra_credentials_list' % camelcase_to_underscore(obj.__class__.__name__) + res['extra_credentials'] = self.reverse(view, kwargs={'pk': obj.pk}) + else: + cloud_cred = obj.cloud_credential + if cloud_cred: + res['cloud_credential'] = self.reverse('api:credential_detail', kwargs={'pk': cloud_cred}) + net_cred = obj.network_credential + if net_cred: + res['cloud_credential'] = self.reverse('api:credential_detail', kwargs={'pk': net_cred}) + return res def to_representation(self, obj): @@ -2142,13 +2180,38 @@ class JobOptionsSerializer(LabelsListMixin, BaseSerializer): ret['playbook'] = '' if 'credential' in ret and not obj.credential: ret['credential'] = None - if 'cloud_credential' in ret and not obj.cloud_credential: - ret['cloud_credential'] = None - if 'network_credential' in ret and not obj.network_credential: - ret['network_credential'] = None + if 'vault_credential' in ret and not obj.vault_credential: + ret['vault_credential'] = None + if self.version == 1: + ret['cloud_credential'] = obj.cloud_credential + ret['network_credential'] = obj.network_credential return ret def validate(self, attrs): + if self.version == 1: # TODO: remove in 3.3 + if 'cloud_credential' in attrs: + pk = attrs.pop('cloud_credential') + for cred in self.instance.cloud_credentials: + self.instance.extra_credentials.remove(cred) + if pk: + cred = Credential.objects.get(pk=pk) + if cred.credential_type.kind != 'cloud': + raise serializers.ValidationError({ + 'cloud_credential': _('You must provide a cloud credential.'), + }) + self.instance.extra_credentials.add(cred) + if 'network_credential' in attrs: + pk = attrs.pop('network_credential') + for cred in self.instance.network_credentials: + self.instance.extra_credentials.remove(cred) + if pk: + cred = Credential.objects.get(pk=pk) + if cred.credential_type.kind != 'net': + raise serializers.ValidationError({ + 'network_credential': _('You must provide a network credential.'), + }) + self.instance.extra_credentials.add(cred) + if 'project' in self.fields and 'playbook' in self.fields: project = attrs.get('project', self.instance and self.instance.project or None) playbook = attrs.get('playbook', self.instance and self.instance.playbook or '') @@ -2309,10 +2372,6 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer): data.setdefault('playbook', job_template.playbook) if job_template.credential: data.setdefault('credential', job_template.credential.pk) - if job_template.cloud_credential: - data.setdefault('cloud_credential', job_template.cloud_credential.pk) - if job_template.network_credential: - data.setdefault('network_credential', job_template.network_credential.pk) data.setdefault('forks', job_template.forks) data.setdefault('limit', job_template.limit) data.setdefault('verbosity', job_template.verbosity) diff --git a/awx/api/urls.py b/awx/api/urls.py index eb7e331cee..d5e102082e 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -385,7 +385,9 @@ v1_urls = patterns('awx.api.views', v2_urls = patterns('awx.api.views', url(r'^$', 'api_v2_root_view'), url(r'^credential_types/', include(credential_type_urls)), - url(r'^hosts/(?P[0-9]+)/ansible_facts/$', 'host_ansible_facts_detail'), + url(r'^hosts/(?P[0-9]+)/ansible_facts/$', 'host_ansible_facts_detail'), + url(r'^jobs/(?P[0-9]+)/extra_credentials/$', 'job_extra_credentials_list'), + url(r'^job_templates/(?P[0-9]+)/extra_credentials/$', 'job_template_extra_credentials_list'), ) urlpatterns = patterns('awx.api.views', diff --git a/awx/api/views.py b/awx/api/views.py index c496f99d9f..debfea42ae 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2467,7 +2467,7 @@ class JobTemplateList(ListCreateAPIView): always_allow_superuser = False capabilities_prefetch = [ 'admin', 'execute', - {'copy': ['project.use', 'inventory.use', 'credential.use', 'cloud_credential.use', 'network_credential.use']} + {'copy': ['project.use', 'inventory.use', 'credential.use']} ] def post(self, request, *args, **kwargs): @@ -2694,6 +2694,21 @@ class JobTemplateNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIVi new_in_300 = True +class JobTemplateExtraCredentialsList(SubListCreateAttachDetachAPIView): + + model = Credential + serializer_class = CredentialSerializer + parent_model = JobTemplate + relationship = 'extra_credentials' + new_in_320 = True + new_in_api_v2 = True + + def is_valid_relation(self, parent, sub, created=False): + if sub.credential_type.kind not in ('net', 'cloud'): + return {'error': _('Extra credentials must be network or cloud.')} + return super(JobTemplateExtraCredentialsList, self).is_valid_relation(parent, sub, created) + + class JobTemplateLabelList(DeleteLastUnattachLabelMixin, SubListCreateAttachDetachAPIView): model = Label @@ -3455,6 +3470,21 @@ class JobDetail(RetrieveUpdateDestroyAPIView): return super(JobDetail, self).destroy(request, *args, **kwargs) +class JobExtraCredentialsList(SubListCreateAttachDetachAPIView): + + model = Credential + serializer_class = CredentialSerializer + parent_model = Job + relationship = 'extra_credentials' + new_in_320 = True + new_in_api_v2 = True + + def is_valid_relation(self, parent, sub, created=False): + if sub.credential_type.kind not in ('net', 'cloud'): + return {'error': _('Extra credentials must be network or cloud.')} + return super(JobExtraCredentialsList, self).is_valid_relation(parent, sub, created) + + class JobLabelList(SubListAPIView): model = Label @@ -3941,7 +3971,6 @@ class UnifiedJobTemplateList(ListAPIView): capabilities_prefetch = [ 'admin', 'execute', {'copy': ['jobtemplate.project.use', 'jobtemplate.inventory.use', 'jobtemplate.credential.use', - 'jobtemplate.cloud_credential.use', 'jobtemplate.network_credential.use', 'workflowjobtemplate.organization.admin']} ] diff --git a/awx/main/access.py b/awx/main/access.py index de118499c5..ed5fd5c90d 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -821,14 +821,10 @@ class CredentialTypeAccess(BaseAccess): def can_use(self, obj): return True - def can_add(self, data): - return self.user.is_superuser - - def can_change(self, obj, data): - return self.user.is_superuser and not obj.managed_by_tower - - def can_delete(self, obj): - return self.user.is_superuser and not obj.managed_by_tower + def get_method_capability(self, method, obj, parent_obj): + if obj.managed_by_tower: + return False + return super(CredentialTypeAccess, self).get_method_capability(method, obj, parent_obj) class CredentialAccess(BaseAccess): @@ -1072,7 +1068,7 @@ class JobTemplateAccess(BaseAccess): else: qs = self.model.accessible_objects(self.user, 'read_role') return qs.select_related('created_by', 'modified_by', 'inventory', 'project', - 'credential', 'cloud_credential', 'next_schedule').all() + 'credential', 'next_schedule').all() def can_add(self, data): ''' @@ -1113,13 +1109,8 @@ class JobTemplateAccess(BaseAccess): if not self.check_related('credential', Credential, data, role_field='use_role'): return False - # If a cloud credential is provided, the user should have use access. - if not self.check_related('cloud_credential', Credential, data, role_field='use_role'): - return False - - # If a network credential is provided, the user should have use access. - if not self.check_related('network_credential', Credential, data, role_field='use_role'): - return False + # TODO: If a vault credential is provided, the user should have use access to it. + # TODO: If any credential in extra_credentials, the user must have access # If an inventory is provided, the user should have use access. inventory = get_value(Inventory, 'inventory') @@ -1185,7 +1176,8 @@ class JobTemplateAccess(BaseAccess): self.check_license(feature='surveys') return True - for required_field in ('credential', 'cloud_credential', 'network_credential', 'inventory', 'project'): + # TODO: handle vault_credential and extra_credentials + for required_field in ('credential', 'inventory', 'project'): required_obj = getattr(obj, required_field, None) if required_field not in data_for_change and required_obj is not None: data_for_change[required_field] = required_obj.pk @@ -1219,8 +1211,6 @@ class JobTemplateAccess(BaseAccess): project_id = data.get('project', obj.project.id if obj.project else None) inventory_id = data.get('inventory', obj.inventory.id if obj.inventory else None) credential_id = data.get('credential', obj.credential.id if obj.credential else None) - cloud_credential_id = data.get('cloud_credential', obj.cloud_credential.id if obj.cloud_credential else None) - network_credential_id = data.get('network_credential', obj.network_credential.id if obj.network_credential else None) if project_id and self.user not in Project.objects.get(pk=project_id).use_role: return False @@ -1228,10 +1218,7 @@ class JobTemplateAccess(BaseAccess): return False if credential_id and self.user not in Credential.objects.get(pk=credential_id).use_role: return False - if cloud_credential_id and self.user not in Credential.objects.get(pk=cloud_credential_id).use_role: - return False - if network_credential_id and self.user not in Credential.objects.get(pk=network_credential_id).use_role: - return False + # TODO: handle vault_credential and extra_credentials return True @@ -1271,7 +1258,7 @@ class JobAccess(BaseAccess): def get_queryset(self): qs = self.model.objects qs = qs.select_related('created_by', 'modified_by', 'job_template', 'inventory', - 'project', 'credential', 'cloud_credential', 'job_template') + 'project', 'credential', 'job_template') qs = qs.prefetch_related('unified_job_template') if self.user.is_superuser or self.user.is_system_auditor: return qs.all() @@ -1907,7 +1894,6 @@ class UnifiedJobTemplateAccess(BaseAccess): # 'project', # 'inventory', # 'credential', - # 'cloud_credential', #) return qs.all() @@ -1957,14 +1943,12 @@ class UnifiedJobAccess(BaseAccess): # 'credential', # 'job_template', # 'inventory_source', - # 'cloud_credential', # 'project___credential', # 'inventory_source___credential', # 'inventory_source___inventory', # 'job_template__inventory', # 'job_template__project', # 'job_template__credential', - # 'job_template__cloud_credential', #) return qs.all() @@ -2150,7 +2134,7 @@ class ActivityStreamAccess(BaseAccess): ''' qs = self.model.objects.all() qs = qs.prefetch_related('organization', 'user', 'inventory', 'host', 'group', 'inventory_source', - 'inventory_update', 'credential', 'team', 'project', 'project_update', + 'inventory_update', 'credential', 'credential_type', 'team', 'project', 'project_update', 'job_template', 'job', 'ad_hoc_command', 'notification_template', 'notification', 'label', 'role', 'actor', 'schedule', 'custom_inventory_script', 'unified_job_template', diff --git a/awx/main/migrations/0040_v320_add_credentialtype_model.py b/awx/main/migrations/0040_v320_add_credentialtype_model.py index fa335d8e3c..da3b4de342 100644 --- a/awx/main/migrations/0040_v320_add_credentialtype_model.py +++ b/awx/main/migrations/0040_v320_add_credentialtype_model.py @@ -64,4 +64,21 @@ class Migration(migrations.Migration): name='credential', unique_together=set([('organization', 'name', 'credential_type')]), ), + + # Connecting activity stream + migrations.AddField( + model_name='activitystream', + name='credential_type', + field=models.ManyToManyField(to='main.CredentialType', blank=True), + ), + migrations.AlterField( + model_name='credential', + name='credential_type', + field=models.ForeignKey(related_name='credentials', to='main.CredentialType', help_text='Type for this credential. Credential Types define valid fields (e.g,. "username", "password") and their properties (e.g,. "username is required" or "password should be stored with encryption").'), + ), + migrations.AlterField( + model_name='credential', + name='inputs', + field=awx.main.fields.CredentialInputField(default={}, help_text='Data structure used to specify input values (e.g., {"username": "jane-doe", "password": "secret"}). Valid fields and their requirements vary depending on the fields defined on the chosen CredentialType.', blank=True), + ), ] diff --git a/awx/main/migrations/0043_v320_job_template_multi_credential.py b/awx/main/migrations/0043_v320_job_template_multi_credential.py new file mode 100644 index 0000000000..ff06b44228 --- /dev/null +++ b/awx/main/migrations/0043_v320_job_template_multi_credential.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +from awx.main.migrations import _credentialtypes as credentialtypes +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0042_v320_drop_v1_credential_fields'), + ] + + operations = [ + migrations.AddField( + model_name='job', + name='extra_credentials', + field=models.ManyToManyField(related_name='_job_extra_credentials_+', to='main.Credential'), + ), + migrations.AddField( + model_name='jobtemplate', + name='extra_credentials', + field=models.ManyToManyField(related_name='_jobtemplate_extra_credentials_+', to='main.Credential'), + ), + migrations.RunPython(credentialtypes.migrate_job_credentials), + migrations.RemoveField( + model_name='job', + name='cloud_credential', + ), + migrations.RemoveField( + model_name='job', + name='network_credential', + ), + migrations.RemoveField( + model_name='jobtemplate', + name='cloud_credential', + ), + migrations.RemoveField( + model_name='jobtemplate', + name='network_credential', + ), + ] diff --git a/awx/main/migrations/_credentialtypes.py b/awx/main/migrations/_credentialtypes.py index 3e91dc3fd3..7aac3eb4b9 100644 --- a/awx/main/migrations/_credentialtypes.py +++ b/awx/main/migrations/_credentialtypes.py @@ -81,3 +81,21 @@ def migrate_to_v2_credentials(apps, schema_editor): new_cred.save() finally: utils.get_current_apps = orig_current_apps + + +def migrate_job_credentials(apps, schema_editor): + # this monkey-patch is necessary to make the implicit role generation save + # signal use the correct Role model (the version active at this point in + # migration, not the one at HEAD) + orig_current_apps = utils.get_current_apps + try: + utils.get_current_apps = lambda: apps + for type_ in ('Job', 'JobTemplate'): + for obj in apps.get_model('main', type_).objects.all(): + if obj.cloud_credential: + obj.extra_credentials.add(obj.cloud_credential) + if obj.network_credential: + obj.extra_credentials.add(obj.network_credential) + obj.save() + finally: + utils.get_current_apps = orig_current_apps diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index c4623571ac..646618d1e8 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -109,6 +109,7 @@ activity_stream_registrar.connect(Group) activity_stream_registrar.connect(InventorySource) #activity_stream_registrar.connect(InventoryUpdate) activity_stream_registrar.connect(Credential) +activity_stream_registrar.connect(CredentialType) activity_stream_registrar.connect(Team) activity_stream_registrar.connect(Project) #activity_stream_registrar.connect(ProjectUpdate) diff --git a/awx/main/models/activity_stream.py b/awx/main/models/activity_stream.py index ddbdae1227..0e78cc2d56 100644 --- a/awx/main/models/activity_stream.py +++ b/awx/main/models/activity_stream.py @@ -45,6 +45,7 @@ class ActivityStream(models.Model): inventory_source = models.ManyToManyField("InventorySource", blank=True) inventory_update = models.ManyToManyField("InventoryUpdate", blank=True) credential = models.ManyToManyField("Credential", blank=True) + credential_type = models.ManyToManyField("CredentialType", blank=True) team = models.ManyToManyField("Team", blank=True) project = models.ManyToManyField("Project", blank=True) project_update = models.ManyToManyField("ProjectUpdate", blank=True) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index c15ab663a3..40f4c22e9a 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -21,7 +21,6 @@ from django.core.exceptions import ValidationError # AWX from awx.api.versioning import reverse -from awx.main.constants import CLOUD_PROVIDERS from awx.main.models.base import * # noqa from awx.main.models.unified_jobs import * # noqa from awx.main.models.notifications import ( @@ -96,21 +95,9 @@ class JobOptions(BaseModel): default=None, on_delete=models.SET_NULL, ) - cloud_credential = models.ForeignKey( + extra_credentials = models.ManyToManyField( 'Credential', - related_name='%(class)ss_as_cloud_credential+', - blank=True, - null=True, - default=None, - on_delete=models.SET_NULL, - ) - network_credential = models.ForeignKey( - 'Credential', - related_name='%(class)ss_as_network_credential+', - blank=True, - null=True, - default=None, - on_delete=models.SET_NULL, + related_name='%(class)ss_as_extra_credential+', ) forks = models.PositiveIntegerField( blank=True, @@ -170,26 +157,60 @@ class JobOptions(BaseModel): cred = self.credential if cred and cred.kind != 'ssh': raise ValidationError( - _('You must provide a machine / SSH credential.'), + _('You must provide an SSH credential.'), ) return cred - def clean_network_credential(self): - cred = self.network_credential - if cred and cred.kind != 'net': + def clean_vault_credential(self): + cred = self.vault_credential + if cred and cred.kind != 'vault': raise ValidationError( - _('You must provide a network credential.'), + _('You must provide a Vault credential.'), ) return cred - def clean_cloud_credential(self): - cred = self.cloud_credential - if cred and cred.kind not in CLOUD_PROVIDERS + ('aws',): - raise ValidationError( - _('Must provide a credential for a cloud provider, such as ' - 'Amazon Web Services or Rackspace.'), - ) - return cred + def clean(self): + super(JobOptions, self).clean() + # extra_credentials M2M can't be accessed until a primary key exists + if self.pk: + for cred in self.extra_credentials.all(): + if cred.credential_type.kind not in ('net', 'cloud'): + raise ValidationError( + _('Extra credentials must be network or cloud.'), + ) + + @property + def all_credentials(self): + credentials = self.extra_credentials.all() + if self.vault_credential: + credentials.insert(0, self.vault_credential) + if self.credential: + credentials.insert(0, self.credential) + return credentials + + @property + def network_credentials(self): + return [cred for cred in self.extra_credentials.all() if cred.credential_type.kind == 'net'] + + @property + def cloud_credentials(self): + return [cred for cred in self.extra_credentials.all() if cred.credential_type.kind == 'cloud'] + + # TODO: remove when API v1 is removed + @property + def cloud_credential(self): + try: + return self.cloud_credentials[-1].pk + except IndexError: + return None + + # TODO: remove when API v1 is removed + @property + def network_credential(self): + try: + return self.network_credentials[-1].pk + except IndexError: + return None @property def passwords_needed_to_start(self): @@ -262,11 +283,11 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour @classmethod def _get_unified_job_field_names(cls): return ['name', 'description', 'job_type', 'inventory', 'project', - 'playbook', 'credential', 'cloud_credential', 'network_credential', 'forks', 'schedule', - 'limit', 'verbosity', 'job_tags', 'extra_vars', 'launch_type', - 'force_handlers', 'skip_tags', 'start_at_task', 'become_enabled', - 'labels', 'survey_passwords', 'allow_simultaneous', 'timeout', - 'store_facts',] + 'playbook', 'credential', 'extra_credentials', 'forks', + 'schedule', 'limit', 'verbosity', 'job_tags', 'extra_vars', + 'launch_type', 'force_handlers', 'skip_tags', 'start_at_task', + 'become_enabled', 'labels', 'survey_passwords', + 'allow_simultaneous', 'timeout', 'store_facts',] def resource_validation_data(self): ''' diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 3104527b96..dadba23108 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -397,31 +397,40 @@ class BaseTask(Task): def build_private_data_files(self, instance, **kwargs): ''' - Create a temporary files containing the private data. - Returns a dictionary with keys from build_private_data - (i.e. 'credential', 'cloud_credential', 'network_credential') and values the file path. + Creates temporary files containing the private data. + Returns a dictionary i.e., + + { + 'credentials': { + : '/path/to/decrypted/data', + : '/path/to/decrypted/data', + : '/path/to/decrypted/data', + } + } ''' private_data = self.build_private_data(instance, **kwargs) - private_data_files = {} + private_data_files = {'credentials': {}} if private_data is not None: ssh_ver = get_ssh_version() ssh_too_old = True if ssh_ver == "unknown" else Version(ssh_ver) < Version("6.0") openssh_keys_supported = ssh_ver != "unknown" and Version(ssh_ver) >= Version("6.5") - for name, data in private_data.iteritems(): + for credential, data in private_data.get('credentials', {}).iteritems(): # Bail out now if a private key was provided in OpenSSH format # and we're running an earlier version (<6.5). if 'OPENSSH PRIVATE KEY' in data and not openssh_keys_supported: raise RuntimeError(OPENSSH_KEY_ERROR) - for name, data in private_data.iteritems(): + for credential, data in private_data.get('credentials', {}).iteritems(): + name = 'credential_%d' % credential.pk # OpenSSH formatted keys must have a trailing newline to be # accepted by ssh-add. if 'OPENSSH PRIVATE KEY' in data and not data.endswith('\n'): data += '\n' # For credentials used with ssh-add, write to a named pipe which # will be read then closed, instead of leaving the SSH key on disk. - if name in ('credential', 'scm_credential', 'ad_hoc_credential') and not ssh_too_old: + if credential.kind in ('ssh', 'scm') and not ssh_too_old: path = os.path.join(kwargs.get('private_data_dir', tempfile.gettempdir()), name) self.open_fifo_write(path, data) + private_data_files['credentials']['ssh'] = path # Ansible network modules do not yet support ssh-agent. # Instead, ssh private key file is explicitly passed via an # env variable. @@ -431,7 +440,7 @@ class BaseTask(Task): f.write(data) f.close() os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) - private_data_files[name] = path + private_data_files['credentials'][credential] = path return private_data_files def open_fifo_write(self, path, data): @@ -515,12 +524,6 @@ class BaseTask(Task): def args2cmdline(self, *args): return ' '.join([pipes.quote(a) for a in args]) - def get_ssh_key_path(self, instance, **kwargs): - ''' - Return the path to the SSH key file, if present. - ''' - return '' - def wrap_args_with_ssh_agent(self, args, ssh_key_path, ssh_auth_sock=None): if ssh_key_path: cmd = ' && '.join([self.args2cmdline('ssh-add', ssh_key_path), @@ -724,8 +727,11 @@ class BaseTask(Task): safe_env = self.build_safe_env(env, **kwargs) # handle custom injectors specified on the CredentialType - for type_ in ('credential', 'cloud_credential', 'network_credential'): - credential = getattr(instance, type_, None) + if hasattr(instance, 'all_credentials'): + credentials = instance.all_credentials + else: + credentials = [instance.credential] + for credential in credentials: if credential: credential.credential_type.inject_credential( credential, env, safe_env, args, safe_args, kwargs['private_data_dir'] @@ -788,6 +794,22 @@ class BaseTask(Task): if not hasattr(settings, 'CELERY_UNIT_TEST'): self.signal_finished(pk) + def get_ssh_key_path(self, instance, **kwargs): + ''' + If using an SSH key, return the path for use by ssh-agent. + ''' + private_data_files = kwargs.get('private_data_files', {}) + if 'ssh' in private_data_files.get('credentials', {}): + return private_data_files['credentials']['ssh'] + ''' + Note: Don't inject network ssh key data into ssh-agent for network + credentials because the ansible modules do not yet support it. + We will want to add back in support when/if Ansible network modules + support this. + ''' + + return '' + class RunJob(BaseTask): ''' @@ -800,36 +822,36 @@ class RunJob(BaseTask): def build_private_data(self, job, **kwargs): ''' Returns a dict of the form - dict['credential'] = - dict['cloud_credential'] = - dict['network_credential'] = - ''' - job_credentials = ['credential', 'cloud_credential', 'network_credential'] - private_data = {} - # If we were sent SSH credentials, decrypt them and send them - # back (they will be written to a temporary file). - for cred_name in job_credentials: - credential = getattr(job, cred_name, None) - if credential: - if credential.ssh_key_data not in (None, ''): - private_data[cred_name] = decrypt_field(credential, 'ssh_key_data') or '' - - if job.cloud_credential and job.cloud_credential.kind == 'openstack': - credential = job.cloud_credential - openstack_auth = dict(auth_url=credential.host, - username=credential.username, - password=decrypt_field(credential, "password"), - project_name=credential.project) - if credential.domain not in (None, ''): - openstack_auth['domain_name'] = credential.domain - openstack_data = { - 'clouds': { - 'devstack': { - 'auth': openstack_auth, - }, - }, + { + 'credentials': { + : , + : , + : } - private_data['cloud_credential'] = yaml.safe_dump(openstack_data, default_flow_style=False, allow_unicode=True) + } + ''' + private_data = {'credentials': {}} + for credential in job.all_credentials: + # If we were sent SSH credentials, decrypt them and send them + # back (they will be written to a temporary file). + if credential.ssh_key_data not in (None, ''): + private_data['credentials'][credential] = decrypt_field(credential, 'ssh_key_data') or '' + + if credential.kind == 'openstack': + openstack_auth = dict(auth_url=credential.host, + username=credential.username, + password=decrypt_field(credential, "password"), + project_name=credential.project) + if credential.domain not in (None, ''): + openstack_auth['domain_name'] = credential.domain + openstack_data = { + 'clouds': { + 'devstack': { + 'auth': openstack_auth, + }, + }, + } + private_data['credentials'][credential] = yaml.safe_dump(openstack_data, default_flow_style=False, allow_unicode=True) return private_data @@ -894,47 +916,47 @@ class RunJob(BaseTask): env['INVENTORY_HOSTVARS'] = str(True) # Set environment variables for cloud credentials. - cloud_cred = job.cloud_credential - if cloud_cred and cloud_cred.kind == 'aws': - env['AWS_ACCESS_KEY'] = cloud_cred.username - env['AWS_SECRET_KEY'] = decrypt_field(cloud_cred, 'password') - if len(cloud_cred.security_token) > 0: - env['AWS_SECURITY_TOKEN'] = decrypt_field(cloud_cred, 'security_token') - # FIXME: Add EC2_URL, maybe EC2_REGION! - elif cloud_cred and cloud_cred.kind == 'rax': - env['RAX_USERNAME'] = cloud_cred.username - env['RAX_API_KEY'] = decrypt_field(cloud_cred, 'password') - env['CLOUD_VERIFY_SSL'] = str(False) - elif cloud_cred and cloud_cred.kind == 'gce': - env['GCE_EMAIL'] = cloud_cred.username - env['GCE_PROJECT'] = cloud_cred.project - env['GCE_PEM_FILE_PATH'] = kwargs.get('private_data_files', {}).get('cloud_credential', '') - elif cloud_cred and cloud_cred.kind == 'azure': - env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.username - env['AZURE_CERT_PATH'] = kwargs.get('private_data_files', {}).get('cloud_credential', '') - elif cloud_cred and cloud_cred.kind == 'azure_rm': - if len(cloud_cred.client) and len(cloud_cred.tenant): - env['AZURE_CLIENT_ID'] = cloud_cred.client - env['AZURE_SECRET'] = decrypt_field(cloud_cred, 'secret') - env['AZURE_TENANT'] = cloud_cred.tenant - env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.subscription - else: - env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.subscription - env['AZURE_AD_USER'] = cloud_cred.username - env['AZURE_PASSWORD'] = decrypt_field(cloud_cred, 'password') - elif cloud_cred and cloud_cred.kind == 'vmware': - env['VMWARE_USER'] = cloud_cred.username - env['VMWARE_PASSWORD'] = decrypt_field(cloud_cred, 'password') - env['VMWARE_HOST'] = cloud_cred.host - elif cloud_cred and cloud_cred.kind == 'openstack': - env['OS_CLIENT_CONFIG_FILE'] = kwargs.get('private_data_files', {}).get('cloud_credential', '') + cred_files = kwargs.get('private_data_files', {}).get('credentials', {}) + for cloud_cred in job.cloud_credentials: + if cloud_cred and cloud_cred.kind == 'aws': + env['AWS_ACCESS_KEY'] = cloud_cred.username + env['AWS_SECRET_KEY'] = decrypt_field(cloud_cred, 'password') + if len(cloud_cred.security_token) > 0: + env['AWS_SECURITY_TOKEN'] = decrypt_field(cloud_cred, 'security_token') + # FIXME: Add EC2_URL, maybe EC2_REGION! + elif cloud_cred and cloud_cred.kind == 'rax': + env['RAX_USERNAME'] = cloud_cred.username + env['RAX_API_KEY'] = decrypt_field(cloud_cred, 'password') + env['CLOUD_VERIFY_SSL'] = str(False) + elif cloud_cred and cloud_cred.kind == 'gce': + env['GCE_EMAIL'] = cloud_cred.username + env['GCE_PROJECT'] = cloud_cred.project + env['GCE_PEM_FILE_PATH'] = cred_files.get(cloud_cred, '') + elif cloud_cred and cloud_cred.kind == 'azure': + env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.username + env['AZURE_CERT_PATH'] = cred_files.get(cloud_cred, '') + elif cloud_cred and cloud_cred.kind == 'azure_rm': + if len(cloud_cred.client) and len(cloud_cred.tenant): + env['AZURE_CLIENT_ID'] = cloud_cred.client + env['AZURE_SECRET'] = decrypt_field(cloud_cred, 'secret') + env['AZURE_TENANT'] = cloud_cred.tenant + env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.subscription + else: + env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.subscription + env['AZURE_AD_USER'] = cloud_cred.username + env['AZURE_PASSWORD'] = decrypt_field(cloud_cred, 'password') + elif cloud_cred and cloud_cred.kind == 'vmware': + env['VMWARE_USER'] = cloud_cred.username + env['VMWARE_PASSWORD'] = decrypt_field(cloud_cred, 'password') + env['VMWARE_HOST'] = cloud_cred.host + elif cloud_cred and cloud_cred.kind == 'openstack': + env['OS_CLIENT_CONFIG_FILE'] = cred_files.get(cloud_cred, '') - network_cred = job.network_credential - if network_cred: + for network_cred in job.network_credentials: env['ANSIBLE_NET_USERNAME'] = network_cred.username env['ANSIBLE_NET_PASSWORD'] = decrypt_field(network_cred, 'password') - ssh_keyfile = kwargs.get('private_data_files', {}).get('network_credential', '') + ssh_keyfile = cred_files.get(network_cred, '') if ssh_keyfile: env['ANSIBLE_NET_SSH_KEYFILE'] = ssh_keyfile @@ -1099,24 +1121,6 @@ class RunJob(BaseTask): return OutputEventFilter(stdout_handle, job_event_callback) - def get_ssh_key_path(self, instance, **kwargs): - ''' - If using an SSH key, return the path for use by ssh-agent. - ''' - private_data_files = kwargs.get('private_data_files', {}) - if 'credential' in private_data_files: - return private_data_files.get('credential') - ''' - Note: Don't inject network ssh key data into ssh-agent for network - credentials because the ansible modules do not yet support it. - We will want to add back in support when/if Ansible network modules - support this. - ''' - #elif 'network_credential' in private_data_files: - # return private_data_files.get('network_credential') - - return '' - def should_use_proot(self, instance, **kwargs): ''' Return whether this task should use proot. @@ -1169,13 +1173,22 @@ class RunProjectUpdate(BaseTask): def build_private_data(self, project_update, **kwargs): ''' Return SSH private key data needed for this project update. + + Returns a dict of the form + { + 'credentials': { + : , + : , + : + } + } ''' handle, self.revision_path = tempfile.mkstemp() - private_data = {} + private_data = {'credentials': {}} if project_update.credential: credential = project_update.credential if credential.ssh_key_data not in (None, ''): - private_data['scm_credential'] = decrypt_field(project_update.credential, 'ssh_key_data') + private_data['credentials'][credential] = decrypt_field(credential, 'ssh_key_data') return private_data def build_passwords(self, project_update, **kwargs): @@ -1334,12 +1347,6 @@ class RunProjectUpdate(BaseTask): def get_idle_timeout(self): return getattr(settings, 'PROJECT_UPDATE_IDLE_TIMEOUT', None) - def get_ssh_key_path(self, instance, **kwargs): - ''' - If using an SSH key, return the path for use by ssh-agent. - ''' - return kwargs.get('private_data_files', {}).get('scm_credential', '') - def get_stdout_handle(self, instance): stdout_handle = super(RunProjectUpdate, self).get_stdout_handle(instance) @@ -1452,13 +1459,26 @@ class RunInventoryUpdate(BaseTask): model = InventoryUpdate def build_private_data(self, inventory_update, **kwargs): - """Return private data needed for inventory update. + """ + Return private data needed for inventory update. + + Returns a dict of the form + { + 'credentials': { + : , + : , + : + } + } + If no private data is needed, return None. """ + private_data = {'credentials': {}} # If this is Microsoft Azure or GCE, return the RSA key if inventory_update.source in ('azure', 'gce'): credential = inventory_update.credential - return dict(cloud_credential=decrypt_field(credential, 'ssh_key_data')) + private_data['credentials'][credential] = decrypt_field(credential, 'ssh_key_data') + return private_data if inventory_update.source == 'openstack': credential = inventory_update.credential @@ -1486,7 +1506,10 @@ class RunInventoryUpdate(BaseTask): }, 'cache': cache, } - return dict(cloud_credential=yaml.safe_dump(openstack_data, default_flow_style=False, allow_unicode=True)) + private_data['credentials'][credential] = yaml.safe_dump( + openstack_data, default_flow_style=False, allow_unicode=True + ) + return private_data cp = ConfigParser.ConfigParser() # Build custom ec2.ini for ec2 inventory script to use. @@ -1602,7 +1625,8 @@ class RunInventoryUpdate(BaseTask): if cp.sections(): f = cStringIO.StringIO() cp.write(f) - return dict(cloud_credential=f.getvalue()) + private_data['credentials'][inventory_update.credential] = f.getvalue() + return private_data def build_passwords(self, inventory_update, **kwargs): """Build a dictionary of authentication/credential information for @@ -1648,7 +1672,8 @@ class RunInventoryUpdate(BaseTask): # `awx/plugins/inventory` directory; those files should be kept in # sync with those in Ansible core at all times. passwords = kwargs.get('passwords', {}) - cloud_credential = kwargs.get('private_data_files', {}).get('cloud_credential', '') + cred_data = kwargs.get('private_data_files', {}).get('credentials', '') + cloud_credential = cred_data.get(inventory_update.credential, '') if inventory_update.source == 'ec2': if passwords.get('source_username', '') and passwords.get('source_password', ''): env['AWS_ACCESS_KEY_ID'] = passwords['source_username'] @@ -1855,13 +1880,22 @@ class RunAdHocCommand(BaseTask): ''' Return SSH private key data needed for this ad hoc command (only if stored in DB as ssh_key_data). + + Returns a dict of the form + { + 'credentials': { + : , + : , + : + } + } ''' # If we were sent SSH credentials, decrypt them and send them # back (they will be written to a temporary file). creds = ad_hoc_command.credential - private_data = {} + private_data = {'credentials': {}} if creds and creds.ssh_key_data not in (None, ''): - private_data['ad_hoc_credential'] = decrypt_field(creds, 'ssh_key_data') or '' + private_data['credentials'][creds] = decrypt_field(creds, 'ssh_key_data') or '' return private_data def build_passwords(self, ad_hoc_command, **kwargs): @@ -2018,12 +2052,6 @@ class RunAdHocCommand(BaseTask): return OutputEventFilter(stdout_handle, ad_hoc_command_event_callback) - def get_ssh_key_path(self, instance, **kwargs): - ''' - If using an SSH key, return the path for use by ssh-agent. - ''' - return kwargs.get('private_data_files', {}).get('ad_hoc_credential', '') - def should_use_proot(self, instance, **kwargs): ''' Return whether this task should use proot. diff --git a/awx/main/tests/factories/fixtures.py b/awx/main/tests/factories/fixtures.py index 4ec373191f..2f04b70ad4 100644 --- a/awx/main/tests/factories/fixtures.py +++ b/awx/main/tests/factories/fixtures.py @@ -153,9 +153,6 @@ def mk_job_template(name, job_type='run', if jt.credential is None: jt.ask_credential_on_launch = True - jt.network_credential = network_credential - jt.cloud_credential = cloud_credential - jt.project = project jt.survey_spec = spec @@ -164,6 +161,13 @@ def mk_job_template(name, job_type='run', if persisted: jt.save() + if cloud_credential: + cloud_credential.save() + jt.extra_credentials.add(cloud_credential) + if network_credential: + network_credential.save() + jt.extra_credentials.add(network_credential) + jt.save() return jt diff --git a/awx/main/tests/functional/api/test_job.py b/awx/main/tests/functional/api/test_job.py new file mode 100644 index 0000000000..0ca4fcf13b --- /dev/null +++ b/awx/main/tests/functional/api/test_job.py @@ -0,0 +1,88 @@ +import pytest + +from awx.api.versioning import reverse + + +# TODO: test this with RBAC and lower-priveleged users +@pytest.mark.django_db +def test_extra_credential_creation(get, post, organization_factory, job_template_factory, credentialtype_aws): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + job = jt.create_unified_job() + + url = reverse('api:job_extra_credentials_list', kwargs={'version': 'v2', 'pk': job.pk}) + response = post(url, { + 'name': 'My Cred', + 'credential_type': credentialtype_aws.pk, + 'inputs': { + 'username': 'bob', + 'password': 'secret', + } + }, objs.superusers.admin) + assert response.status_code == 201 + + response = get(url, user=objs.superusers.admin) + assert response.data.get('count') == 1 + + +# TODO: test this with RBAC and lower-priveleged users +@pytest.mark.django_db +def test_attach_extra_credential(get, post, organization_factory, job_template_factory, credential): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + job = jt.create_unified_job() + + url = reverse('api:job_extra_credentials_list', kwargs={'version': 'v2', 'pk': job.pk}) + response = get(url, user=objs.superusers.admin) + assert response.data.get('count') == 0 + + response = post(url, { + 'associate': True, + 'id': credential.id, + }, objs.superusers.admin) + assert response.status_code == 204 + + response = get(url, user=objs.superusers.admin) + assert response.data.get('count') == 1 + + +# TODO: test this with RBAC and lower-priveleged users +@pytest.mark.django_db +def test_detach_extra_credential(get, post, organization_factory, job_template_factory, credential): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + jt.extra_credentials.add(credential) + jt.save() + job = jt.create_unified_job() + + url = reverse('api:job_extra_credentials_list', kwargs={'version': 'v2', 'pk': job.pk}) + response = get(url, user=objs.superusers.admin) + assert response.data.get('count') == 1 + + response = post(url, { + 'disassociate': True, + 'id': credential.id, + }, objs.superusers.admin) + assert response.status_code == 204 + + response = get(url, user=objs.superusers.admin) + assert response.data.get('count') == 0 + + +@pytest.mark.django_db +def test_attach_extra_credential_wrong_kind_xfail(get, post, organization_factory, job_template_factory, machine_credential): + """Extra credentials only allow net + cloud credentials""" + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + job = jt.create_unified_job() + + url = reverse('api:job_extra_credentials_list', kwargs={'version': 'v2', 'pk': job.pk}) + response = post(url, { + 'associate': True, + 'id': machine_credential.id, + }, objs.superusers.admin) + assert response.status_code == 400 diff --git a/awx/main/tests/functional/api/test_job_template.py b/awx/main/tests/functional/api/test_job_template.py index 09e8be11dd..5cd43bb4b0 100644 --- a/awx/main/tests/functional/api/test_job_template.py +++ b/awx/main/tests/functional/api/test_job_template.py @@ -36,6 +36,133 @@ def test_create(post, project, machine_credential, inventory, alice, grant_proje }, alice, expect=expect) +# TODO: test this with RBAC and lower-priveleged users +@pytest.mark.django_db +def test_extra_credential_creation(get, post, organization_factory, job_template_factory, credentialtype_aws): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + + url = reverse('api:job_template_extra_credentials_list', kwargs={'version': 'v2', 'pk': jt.pk}) + response = post(url, { + 'name': 'My Cred', + 'credential_type': credentialtype_aws.pk, + 'inputs': { + 'username': 'bob', + 'password': 'secret', + } + }, objs.superusers.admin) + assert response.status_code == 201 + + response = get(url, user=objs.superusers.admin) + assert response.data.get('count') == 1 + + +# TODO: test this with RBAC and lower-priveleged users +@pytest.mark.django_db +def test_attach_extra_credential(get, post, organization_factory, job_template_factory, credential): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + + url = reverse('api:job_template_extra_credentials_list', kwargs={'version': 'v2', 'pk': jt.pk}) + response = post(url, { + 'associate': True, + 'id': credential.id, + }, objs.superusers.admin) + assert response.status_code == 204 + + response = get(url, user=objs.superusers.admin) + assert response.data.get('count') == 1 + + +# TODO: test this with RBAC and lower-priveleged users +@pytest.mark.django_db +def test_detach_extra_credential(get, post, organization_factory, job_template_factory, credential): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + jt.extra_credentials.add(credential) + jt.save() + + url = reverse('api:job_template_extra_credentials_list', kwargs={'version': 'v2', 'pk': jt.pk}) + response = post(url, { + 'disassociate': True, + 'id': credential.id, + }, objs.superusers.admin) + assert response.status_code == 204 + + response = get(url, user=objs.superusers.admin) + assert response.data.get('count') == 0 + + +@pytest.mark.django_db +def test_attach_extra_credential_wrong_kind_xfail(get, post, organization_factory, job_template_factory, machine_credential): + """Extra credentials only allow net + cloud credentials""" + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + + url = reverse('api:job_template_extra_credentials_list', kwargs={'version': 'v2', 'pk': jt.pk}) + response = post(url, { + 'associate': True, + 'id': machine_credential.id, + }, objs.superusers.admin) + assert response.status_code == 400 + + response = get(url, user=objs.superusers.admin) + assert response.data.get('count') == 0 + + +@pytest.mark.django_db +def test_v1_extra_credentials_detail(get, organization_factory, job_template_factory, credential, net_credential): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + jt.extra_credentials.add(credential) + jt.extra_credentials.add(net_credential) + jt.save() + + url = reverse('api:job_template_detail', kwargs={'version': 'v1', 'pk': jt.pk}) + response = get(url, user=objs.superusers.admin) + assert response.data.get('cloud_credential') == credential.pk + assert response.data.get('network_credential') == net_credential.pk + + +@pytest.mark.django_db +def test_v1_set_extra_credentials(get, patch, organization_factory, job_template_factory, credential, net_credential): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + jt.save() + + url = reverse('api:job_template_detail', kwargs={'version': 'v1', 'pk': jt.pk}) + response = patch(url, { + 'cloud_credential': credential.pk, + 'network_credential': net_credential.pk + }, objs.superusers.admin) + assert response.status_code == 200 + + url = reverse('api:job_template_detail', kwargs={'version': 'v1', 'pk': jt.pk}) + response = get(url, user=objs.superusers.admin) + assert response.status_code == 200 + assert response.data.get('cloud_credential') == credential.pk + assert response.data.get('network_credential') == net_credential.pk + + url = reverse('api:job_template_detail', kwargs={'version': 'v1', 'pk': jt.pk}) + response = patch(url, { + 'cloud_credential': None, + 'network_credential': None, + }, objs.superusers.admin) + assert response.status_code == 200 + + url = reverse('api:job_template_detail', kwargs={'version': 'v1', 'pk': jt.pk}) + response = get(url, user=objs.superusers.admin) + assert response.status_code == 200 + assert response.data.get('cloud_credential') is None + assert response.data.get('network_credential') is None + + @pytest.mark.django_db @pytest.mark.parametrize( "grant_project, grant_credential, grant_inventory, expect", [ diff --git a/awx/main/tests/functional/api/test_rbac_displays.py b/awx/main/tests/functional/api/test_rbac_displays.py index c905c9da35..115cbf997b 100644 --- a/awx/main/tests/functional/api/test_rbac_displays.py +++ b/awx/main/tests/functional/api/test_rbac_displays.py @@ -316,7 +316,6 @@ def test_prefetch_jt_copy_capability(job_template, project, inventory, machine_c qs = JobTemplate.objects.all() cache_list_capabilities(qs, [{'copy': [ 'project.use', 'inventory.use', 'credential.use', - 'cloud_credential.use', 'network_credential.use' ]}], JobTemplate, rando) assert qs[0].capabilities_cache == {'copy': False} @@ -326,7 +325,6 @@ def test_prefetch_jt_copy_capability(job_template, project, inventory, machine_c cache_list_capabilities(qs, [{'copy': [ 'project.use', 'inventory.use', 'credential.use', - 'cloud_credential.use', 'network_credential.use' ]}], JobTemplate, rando) assert qs[0].capabilities_cache == {'copy': True} diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index c6e54773ce..777c5a08e8 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -215,6 +215,12 @@ def credential(credentialtype_aws): inputs={'username': 'something', 'password': 'secret'}) +@pytest.fixture +def net_credential(credentialtype_net): + return Credential.objects.create(credential_type=credentialtype_net, name='test-cred', + inputs={'username': 'something', 'password': 'secret'}) + + @pytest.fixture def machine_credential(credentialtype_ssh): return Credential.objects.create(credential_type=credentialtype_ssh, name='machine-cred', diff --git a/awx/main/tests/functional/models/test_activity_stream.py b/awx/main/tests/functional/models/test_activity_stream.py index b9c1611f05..44c19fc7d8 100644 --- a/awx/main/tests/functional/models/test_activity_stream.py +++ b/awx/main/tests/functional/models/test_activity_stream.py @@ -1,7 +1,15 @@ import pytest +import json + # AWX models -from awx.main.models import ActivityStream, Organization, JobTemplate +from awx.main.models import ( + ActivityStream, + Organization, + JobTemplate, + Credential, + CredentialType +) @@ -80,3 +88,46 @@ class TestRolesAssociationEntries: proj2.use_role.parents.add(proj1.admin_role) assert ActivityStream.objects.filter(role=proj1.admin_role, project=proj2).count() == 1 + + +@pytest.fixture +def somecloud_type(): + return CredentialType.objects.create( + kind='cloud', + name='SomeCloud', + managed_by_tower=False, + inputs={ + 'fields': [{ + 'id': 'api_token', + 'label': 'API Token', + 'type': 'string', + 'secret': True + }] + }, + injectors={ + 'env': { + 'MY_CLOUD_API_TOKEN': '{{api_token.foo()}}' + } + } + ) + + +@pytest.mark.django_db +class TestCredentialModels: + ''' + Assure that core elements of activity stream feature are working + ''' + + def test_create_credential_type(self, somecloud_type): + assert ActivityStream.objects.filter(credential_type=somecloud_type).count() == 1 + entry = ActivityStream.objects.filter(credential_type=somecloud_type)[0] + assert entry.operation == 'create' + + def test_credential_hidden_information(self, somecloud_type): + cred = Credential.objects.create( + credential_type=somecloud_type, + inputs = {'api_token': 'ABC123'} + ) + entry = ActivityStream.objects.filter(credential=cred)[0] + assert entry.operation == 'create' + assert json.loads(entry.changes)['inputs'] == 'hidden' diff --git a/awx/main/tests/functional/models/test_job_options.py b/awx/main/tests/functional/models/test_job_options.py new file mode 100644 index 0000000000..34bb7d7cae --- /dev/null +++ b/awx/main/tests/functional/models/test_job_options.py @@ -0,0 +1,60 @@ +import pytest + +from django.core.exceptions import ValidationError +from awx.main.models import Credential + + +@pytest.mark.django_db +def test_clean_credential_with_ssh_type(credentialtype_ssh, job_template): + credential = Credential( + name='My Credential', + credential_type=credentialtype_ssh + ) + credential.save() + + job_template.credential = credential + job_template.full_clean() + + +@pytest.mark.django_db +def test_clean_credential_with_invalid_type_xfail(credentialtype_aws, job_template): + credential = Credential( + name='My Credential', + credential_type=credentialtype_aws + ) + credential.save() + + with pytest.raises(ValidationError): + job_template.credential = credential + job_template.full_clean() + + +@pytest.mark.django_db +def test_clean_credential_with_custom_types(credentialtype_aws, credentialtype_net, job_template): + aws = Credential( + name='AWS Credential', + credential_type=credentialtype_aws + ) + aws.save() + net = Credential( + name='Net Credential', + credential_type=credentialtype_net + ) + net.save() + + job_template.extra_credentials.add(aws) + job_template.extra_credentials.add(net) + job_template.full_clean() + + +@pytest.mark.django_db +def test_clean_credential_with_custom_types_xfail(credentialtype_ssh, job_template): + ssh = Credential( + name='SSH Credential', + credential_type=credentialtype_ssh + ) + ssh.save() + + with pytest.raises(ValidationError): + job_template.extra_credentials.add(ssh) + job_template.full_clean() diff --git a/awx/main/tests/unit/test_access.py b/awx/main/tests/unit/test_access.py index 05199fd5e3..c0753bb8a3 100644 --- a/awx/main/tests/unit/test_access.py +++ b/awx/main/tests/unit/test_access.py @@ -159,6 +159,7 @@ def test_jt_existing_values_are_nonsensitive(job_template_with_ids, user_unit): assert access.changes_are_non_sensitive(job_template_with_ids, data) +@pytest.mark.xfail # TODO: update this to respect JT.extra_credentials def test_change_jt_sensitive_data(job_template_with_ids, mocker, user_unit): """Assure that can_add is called with all ForeignKeys.""" diff --git a/awx/main/tests/unit/test_network_credential.py b/awx/main/tests/unit/test_network_credential.py deleted file mode 100644 index 7435f86617..0000000000 --- a/awx/main/tests/unit/test_network_credential.py +++ /dev/null @@ -1,133 +0,0 @@ -import pytest - -from awx.main.models.credential import CredentialType, Credential -from awx.main.models.jobs import Job -from awx.main.models.inventory import Inventory -from awx.main.tasks import RunJob - - -def test_aws_cred_parse(mocker): - with mocker.patch('django.db.ConnectionRouter.db_for_write'): - job = Job(id=1) - job.inventory = mocker.MagicMock(spec=Inventory, id=2) - aws = CredentialType.defaults['aws']() - - options = { - 'credential_type': aws, - 'inputs': { - 'username': 'aws_user', - 'password': 'aws_passwd', - 'security_token': 'token', - } - } - job.cloud_credential = Credential(**options) - - run_job = RunJob() - mocker.patch.object(run_job, 'should_use_proot', return_value=False) - - env = run_job.build_env(job, private_data_dir='/tmp') - assert env['AWS_ACCESS_KEY'] == options['inputs']['username'] - assert env['AWS_SECRET_KEY'] == options['inputs']['password'] - assert env['AWS_SECURITY_TOKEN'] == options['inputs']['security_token'] - - -def test_net_cred_parse(mocker): - with mocker.patch('django.db.ConnectionRouter.db_for_write'): - job = Job(id=1) - job.inventory = mocker.MagicMock(spec=Inventory, id=2) - net = CredentialType.defaults['aws']() - - options = { - 'credential_type': net, - 'inputs': { - 'username':'test', - 'password':'test', - 'authorize': True, - 'authorize_password': 'passwd', - 'ssh_key_data': """-----BEGIN PRIVATE KEY-----\nstuff==\n-----END PRIVATE KEY-----""", - } - } - private_data_files = { - 'network_credential': '/tmp/this_file_does_not_exist_during_test_but_the_path_is_real', - } - job.network_credential = Credential(**options) - - run_job = RunJob() - mocker.patch.object(run_job, 'should_use_proot', return_value=False) - - env = run_job.build_env(job, private_data_dir='/tmp', private_data_files=private_data_files) - assert env['ANSIBLE_NET_USERNAME'] == options['inputs']['username'] - assert env['ANSIBLE_NET_PASSWORD'] == options['inputs']['password'] - assert env['ANSIBLE_NET_AUTHORIZE'] == '1' - assert env['ANSIBLE_NET_AUTH_PASS'] == options['inputs']['authorize_password'] - assert env['ANSIBLE_NET_SSH_KEYFILE'] == private_data_files['network_credential'] - - -@pytest.fixture -def mock_job(mocker): - ssh = CredentialType.defaults['ssh']() - options = { - 'credential_type': ssh, - 'inputs': { - 'username':'test', - 'password':'test', - 'ssh_key_data': """-----BEGIN PRIVATE KEY-----\nstuff==\n-----END PRIVATE KEY-----""", - 'authorize': True, - 'authorize_password': 'passwd', - } - } - - mock_job_attrs = {'forks': False, 'id': 1, 'cancel_flag': False, 'status': 'running', 'job_type': 'normal', - 'credential': None, 'cloud_credential': None, 'network_credential': Credential(**options), - 'become_enabled': False, 'become_method': None, 'become_username': None, - 'inventory': mocker.MagicMock(spec=Inventory, id=2), 'force_handlers': False, - 'limit': None, 'verbosity': None, 'job_tags': None, 'skip_tags': None, - 'start_at_task': None, 'pk': 1, 'launch_type': 'normal', 'job_template':None, - 'created_by': None, 'extra_vars_dict': None, 'project':None, 'playbook': 'test.yml', - 'store_facts': False,} - mock_job = mocker.MagicMock(spec=Job, **mock_job_attrs) - return mock_job - - -@pytest.fixture -def run_job_net_cred(mocker, get_ssh_version, mock_job): - mocker.patch('django.db.ConnectionRouter.db_for_write') - run_job = RunJob() - - mocker.patch.object(run_job, 'update_model', return_value=mock_job) - mocker.patch.object(run_job, 'build_cwd', return_value='/tmp') - mocker.patch.object(run_job, 'should_use_proot', return_value=False) - mocker.patch.object(run_job, 'run_pexpect', return_value=('successful', 0)) - mocker.patch.object(run_job, 'open_fifo_write', return_value=None) - mocker.patch.object(run_job, 'post_run_hook', return_value=None) - - return run_job - - -@pytest.mark.skip(reason="Note: Ansible network modules don't yet support ssh-agent added keys.") -def test_net_cred_ssh_agent(run_job_net_cred, mock_job): - run_job = run_job_net_cred - run_job.run(mock_job.id) - - assert run_job.update_model.call_count == 4 - - job_args = run_job.update_model.call_args_list[1][1].get('job_args') - assert 'ssh-add' in job_args - assert 'ssh-agent' in job_args - assert 'network_credential' in job_args - - -def test_net_cred_job_model_env(run_job_net_cred, mock_job): - run_job = run_job_net_cred - run_job.run(mock_job.id) - - assert run_job.update_model.call_count == 4 - - job_args = run_job.update_model.call_args_list[1][1].get('job_env') - assert 'ANSIBLE_NET_USERNAME' in job_args - assert 'ANSIBLE_NET_PASSWORD' in job_args - assert 'ANSIBLE_NET_AUTHORIZE' in job_args - assert 'ANSIBLE_NET_AUTH_PASS' in job_args - assert 'ANSIBLE_NET_SSH_KEYFILE' in job_args - - diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 597b7aaa38..ed945cb0fd 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -126,7 +126,9 @@ def test_openstack_client_config_generation(mocker): 'source_vars_dict': {} }) cloud_config = update.build_private_data(inventory_update) - cloud_credential = yaml.load(cloud_config['cloud_credential']) + cloud_credential = yaml.load( + cloud_config.get('credentials')[inventory_update.credential] + ) assert cloud_credential['clouds'] == { 'devstack': { 'auth': { @@ -155,7 +157,9 @@ def test_openstack_client_config_generation_with_private_source_vars(mocker, sou 'source_vars_dict': {'private': source} }) cloud_config = update.build_private_data(inventory_update) - cloud_credential = yaml.load(cloud_config['cloud_credential']) + cloud_credential = yaml.load( + cloud_config.get('credentials')[inventory_update.credential] + ) assert cloud_credential['clouds'] == { 'devstack': { 'auth': { @@ -227,18 +231,27 @@ class TestJobExecution: p.stop() def get_instance(self): - return Job( + job = Job( pk=1, created=datetime.utcnow(), status='new', job_type='run', cancel_flag=False, - credential=None, - cloud_credential=None, - network_credential=None, project=Project() ) + # mock the job.extra_credentials M2M relation so we can avoid DB access + job._extra_credentials = [] + patch = mock.patch.object(Job, 'extra_credentials', mock.Mock( + all=lambda: job._extra_credentials, + add=job._extra_credentials.append, + spec_set=['all', 'add'] + )) + self.patches.append(patch) + patch.start() + + return job + @property def pk(self): return self.instance.pk @@ -278,13 +291,13 @@ class TestJobCredentials(TestJobExecution): def test_ssh_passwords(self, field, password_name, expected_flag): ssh = CredentialType.defaults['ssh']() - self.instance.credential = Credential( + credential = Credential( + pk=1, credential_type=ssh, inputs = {'username': 'bob', field: 'secret'} ) - self.instance.credential.inputs[field] = encrypt_field( - self.instance.credential, field - ) + credential.inputs[field] = encrypt_field(credential, field) + self.instance.credential = credential self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -298,20 +311,20 @@ class TestJobCredentials(TestJobExecution): def test_ssh_key_with_agent(self): ssh = CredentialType.defaults['ssh']() - self.instance.credential = Credential( + credential = Credential( + pk=1, credential_type=ssh, inputs = { 'username': 'bob', 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY } ) - self.instance.credential.inputs['ssh_key_data'] = encrypt_field( - self.instance.credential, 'ssh_key_data' - ) + credential.inputs['ssh_key_data'] = encrypt_field(credential, 'ssh_key_data') + self.instance.credential = credential def run_pexpect_side_effect(private_data, *args, **kwargs): job, args, cwd, env, passwords, stdout = args - ssh_key_data_fifo = '/'.join([private_data, 'credential']) + ssh_key_data_fifo = '/'.join([private_data, 'credential_1']) assert open(ssh_key_data_fifo, 'r').read() == self.EXAMPLE_PRIVATE_KEY assert ' '.join(args).startswith( 'ssh-agent -a %s sh -c ssh-add %s && rm -f %s' % ( @@ -331,13 +344,13 @@ class TestJobCredentials(TestJobExecution): def test_aws_cloud_credential(self): aws = CredentialType.defaults['aws']() - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=aws, inputs = {'username': 'bob', 'password': 'secret'} ) - self.instance.cloud_credential.inputs['password'] = encrypt_field( - self.instance.cloud_credential, 'password' - ) + credential.inputs['password'] = encrypt_field(credential, 'password') + self.instance.extra_credentials.add(credential) self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -350,14 +363,14 @@ class TestJobCredentials(TestJobExecution): def test_aws_cloud_credential_with_sts_token(self): aws = CredentialType.defaults['aws']() - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=aws, inputs = {'username': 'bob', 'password': 'secret', 'security_token': 'token'} ) for key in ('password', 'security_token'): - self.instance.cloud_credential.inputs[key] = encrypt_field( - self.instance.cloud_credential, key - ) + credential.inputs[key] = encrypt_field(credential, key) + self.instance.extra_credentials.add(credential) self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -370,13 +383,13 @@ class TestJobCredentials(TestJobExecution): def test_rax_credential(self): rax = CredentialType.defaults['rackspace']() - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=rax, inputs = {'username': 'bob', 'password': 'secret'} ) - self.instance.cloud_credential.inputs['password'] = encrypt_field( - self.instance.cloud_credential, 'password' - ) + credential.inputs['password'] = encrypt_field(credential, 'password') + self.instance.extra_credentials.add(credential) self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -389,7 +402,8 @@ class TestJobCredentials(TestJobExecution): def test_gce_credentials(self): gce = CredentialType.defaults['gce']() - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=gce, inputs = { 'username': 'bob', @@ -397,9 +411,8 @@ class TestJobCredentials(TestJobExecution): 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY } ) - self.instance.cloud_credential.inputs['ssh_key_data'] = encrypt_field( - self.instance.cloud_credential, 'ssh_key_data' - ) + credential.inputs['ssh_key_data'] = encrypt_field(credential, 'ssh_key_data') + self.instance.extra_credentials.add(credential) def run_pexpect_side_effect(*args, **kwargs): job, args, cwd, env, passwords, stdout = args @@ -414,16 +427,16 @@ class TestJobCredentials(TestJobExecution): def test_azure_credentials(self): azure = CredentialType.defaults['azure']() - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=azure, inputs = { 'username': 'bob', 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY } ) - self.instance.cloud_credential.inputs['ssh_key_data'] = encrypt_field( - self.instance.cloud_credential, 'ssh_key_data' - ) + credential.inputs['ssh_key_data'] = encrypt_field(credential, 'ssh_key_data') + self.instance.extra_credentials.add(credential) def run_pexpect_side_effect(*args, **kwargs): job, args, cwd, env, passwords, stdout = args @@ -437,7 +450,8 @@ class TestJobCredentials(TestJobExecution): def test_azure_rm_with_tenant(self): azure = CredentialType.defaults['azure_rm']() - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=azure, inputs = { 'client': 'some-client', @@ -446,9 +460,8 @@ class TestJobCredentials(TestJobExecution): 'subscription': 'some-subscription' } ) - self.instance.cloud_credential.inputs['secret'] = encrypt_field( - self.instance.cloud_credential, 'secret' - ) + credential.inputs['secret'] = encrypt_field(credential, 'secret') + self.instance.extra_credentials.add(credential) self.task.run(self.pk) @@ -463,7 +476,8 @@ class TestJobCredentials(TestJobExecution): def test_azure_rm_with_password(self): azure = CredentialType.defaults['azure_rm']() - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=azure, inputs = { 'subscription': 'some-subscription', @@ -471,9 +485,8 @@ class TestJobCredentials(TestJobExecution): 'password': 'secret' } ) - self.instance.cloud_credential.inputs['password'] = encrypt_field( - self.instance.cloud_credential, 'password' - ) + credential.inputs['password'] = encrypt_field(credential, 'password') + self.instance.extra_credentials.add(credential) self.task.run(self.pk) @@ -487,13 +500,13 @@ class TestJobCredentials(TestJobExecution): def test_vmware_credentials(self): vmware = CredentialType.defaults['vmware']() - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=vmware, inputs = {'username': 'bob', 'password': 'secret', 'host': 'https://example.org'} ) - self.instance.cloud_credential.inputs['password'] = encrypt_field( - self.instance.cloud_credential, 'password' - ) + credential.inputs['password'] = encrypt_field(credential, 'password') + self.instance.extra_credentials.add(credential) self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -506,7 +519,8 @@ class TestJobCredentials(TestJobExecution): def test_openstack_credentials(self): openstack = CredentialType.defaults['openstack']() - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=openstack, inputs = { 'username': 'bob', @@ -515,9 +529,8 @@ class TestJobCredentials(TestJobExecution): 'host': 'https://keystone.example.org' } ) - self.instance.cloud_credential.inputs['password'] = encrypt_field( - self.instance.cloud_credential, 'password' - ) + credential.inputs['password'] = encrypt_field(credential, 'password') + self.instance.extra_credentials.add(credential) def run_pexpect_side_effect(*args, **kwargs): job, args, cwd, env, passwords, stdout = args @@ -539,7 +552,8 @@ class TestJobCredentials(TestJobExecution): def test_net_credentials(self): net = CredentialType.defaults['net']() - self.instance.network_credential = Credential( + credential = Credential( + pk=1, credential_type=net, inputs = { 'username': 'bob', @@ -550,9 +564,8 @@ class TestJobCredentials(TestJobExecution): } ) for field in ('password', 'ssh_key_data', 'authorize_password'): - self.instance.network_credential.inputs[field] = encrypt_field( - self.instance.network_credential, field - ) + credential.inputs[field] = encrypt_field(credential, field) + self.instance.extra_credentials.add(credential) def run_pexpect_side_effect(*args, **kwargs): job, args, cwd, env, passwords, stdout = args @@ -584,10 +597,12 @@ class TestJobCredentials(TestJobExecution): } } ) - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) + self.instance.extra_credentials.add(credential) with pytest.raises(Exception): self.task.run(self.pk) @@ -609,10 +624,12 @@ class TestJobCredentials(TestJobExecution): } } ) - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) + self.instance.extra_credentials.add(credential) self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -639,10 +656,12 @@ class TestJobCredentials(TestJobExecution): } } ) - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) + self.instance.extra_credentials.add(credential) self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -670,13 +689,13 @@ class TestJobCredentials(TestJobExecution): } } ) - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=some_cloud, inputs = {'password': 'SUPER-SECRET-123'} ) - self.instance.cloud_credential.inputs['password'] = encrypt_field( - self.instance.cloud_credential, 'password' - ) + credential.inputs['password'] = encrypt_field(credential, 'password') + self.instance.extra_credentials.add(credential) self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -704,10 +723,12 @@ class TestJobCredentials(TestJobExecution): } } ) - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) + self.instance.extra_credentials.add(credential) self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -738,13 +759,13 @@ class TestJobCredentials(TestJobExecution): } } ) - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=some_cloud, inputs = {'password': 'SUPER-SECRET-123'} ) - self.instance.cloud_credential.inputs['password'] = encrypt_field( - self.instance.cloud_credential, 'password' - ) + credential.inputs['password'] = encrypt_field(credential, 'password') + self.instance.extra_credentials.add(credential) self.task.run(self.pk) assert self.task.run_pexpect.call_count == 1 @@ -775,10 +796,12 @@ class TestJobCredentials(TestJobExecution): } } ) - self.instance.cloud_credential = Credential( + credential = Credential( + pk=1, credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) + self.instance.extra_credentials.add(credential) self.task.run(self.pk) def run_pexpect_side_effect(*args, **kwargs): @@ -789,6 +812,49 @@ class TestJobCredentials(TestJobExecution): self.task.run_pexpect = mock.Mock(side_effect=run_pexpect_side_effect) self.task.run(self.pk) + def test_multi_cloud(self): + gce = CredentialType.defaults['gce']() + gce_credential = Credential( + pk=1, + credential_type=gce, + inputs = { + 'username': 'bob', + 'project': 'some-project', + 'ssh_key_data': 'GCE: %s' % self.EXAMPLE_PRIVATE_KEY + } + ) + gce_credential.inputs['ssh_key_data'] = encrypt_field(gce_credential, 'ssh_key_data') + self.instance.extra_credentials.add(gce_credential) + + azure = CredentialType.defaults['azure']() + azure_credential = Credential( + pk=2, + credential_type=azure, + inputs = { + 'username': 'joe', + 'ssh_key_data': 'AZURE: %s' % self.EXAMPLE_PRIVATE_KEY + } + ) + azure_credential.inputs['ssh_key_data'] = encrypt_field(azure_credential, 'ssh_key_data') + self.instance.extra_credentials.add(azure_credential) + + def run_pexpect_side_effect(*args, **kwargs): + job, args, cwd, env, passwords, stdout = args + + assert env['GCE_EMAIL'] == 'bob' + assert env['GCE_PROJECT'] == 'some-project' + ssh_key_data = env['GCE_PEM_FILE_PATH'] + assert open(ssh_key_data, 'rb').read() == 'GCE: %s' % self.EXAMPLE_PRIVATE_KEY + + assert env['AZURE_SUBSCRIPTION_ID'] == 'joe' + ssh_key_data = env['AZURE_CERT_PATH'] + assert open(ssh_key_data, 'rb').read() == 'AZURE: %s' % self.EXAMPLE_PRIVATE_KEY + + return ['successful', 0] + + self.task.run_pexpect = mock.Mock(side_effect=run_pexpect_side_effect) + self.task.run(self.pk) + class TestProjectUpdateCredentials(TestJobExecution): @@ -817,6 +883,7 @@ class TestProjectUpdateCredentials(TestJobExecution): ssh = CredentialType.defaults['ssh']() self.instance.scm_type = scm_type self.instance.credential = Credential( + pk=1, credential_type=ssh, inputs = {'username': 'bob', 'password': 'secret'} ) @@ -836,6 +903,7 @@ class TestProjectUpdateCredentials(TestJobExecution): ssh = CredentialType.defaults['ssh']() self.instance.scm_type = scm_type self.instance.credential = Credential( + pk=1, credential_type=ssh, inputs = { 'username': 'bob', @@ -848,7 +916,7 @@ class TestProjectUpdateCredentials(TestJobExecution): def run_pexpect_side_effect(private_data, *args, **kwargs): job, args, cwd, env, passwords, stdout = args - ssh_key_data_fifo = '/'.join([private_data, 'scm_credential']) + ssh_key_data_fifo = '/'.join([private_data, 'credential_1']) assert open(ssh_key_data_fifo, 'r').read() == self.EXAMPLE_PRIVATE_KEY assert ' '.join(args).startswith( 'ssh-agent -a %s sh -c ssh-add %s && rm -f %s' % ( @@ -885,6 +953,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): aws = CredentialType.defaults['aws']() self.instance.source = 'ec2' self.instance.credential = Credential( + pk=1, credential_type=aws, inputs = {'username': 'bob', 'password': 'secret'} ) @@ -911,6 +980,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): vmware = CredentialType.defaults['vmware']() self.instance.source = 'vmware' self.instance.credential = Credential( + pk=1, credential_type=vmware, inputs = {'username': 'bob', 'password': 'secret', 'host': 'https://example.org'} ) @@ -935,6 +1005,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): azure = CredentialType.defaults['azure']() self.instance.source = 'azure' self.instance.credential = Credential( + pk=1, credential_type=azure, inputs = { 'username': 'bob', @@ -959,6 +1030,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): gce = CredentialType.defaults['gce']() self.instance.source = 'gce' self.instance.credential = Credential( + pk=1, credential_type=gce, inputs = { 'username': 'bob', @@ -985,6 +1057,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): openstack = CredentialType.defaults['openstack']() self.instance.source = 'openstack' self.instance.credential = Credential( + pk=1, credential_type=openstack, inputs = { 'username': 'bob', @@ -1019,6 +1092,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): satellite6 = CredentialType.defaults['satellite6']() self.instance.source = 'satellite6' self.instance.credential = Credential( + pk=1, credential_type=satellite6, inputs = { 'username': 'bob', @@ -1046,6 +1120,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): cloudforms = CredentialType.defaults['cloudforms']() self.instance.source = 'cloudforms' self.instance.credential = Credential( + pk=1, credential_type=cloudforms, inputs = { 'username': 'bob', diff --git a/docs/custom_credential_types.md b/docs/custom_credential_types.md new file mode 100644 index 0000000000..60b4e3e179 --- /dev/null +++ b/docs/custom_credential_types.md @@ -0,0 +1,316 @@ +Custom Credential Types Overview +================================ + +Prior to Tower 3.2, Tower included bundled credential types, such as +"Machine", "Network", or "Amazon Web Services". In 3.2, we have added support +for custom types so that customers can extend Tower with support for +third-party credential mechanisms. + +Important Changes +----------------- +* Tower has a new top-level resource, ``Credential Type``, which can fall into + one of several categories, or "kinds": + + - SSH + - Vault + - Source Control + - Network + - Cloud + + ``Credential Types`` are composed of a set of field ``inputs`` (for example, + ``"username"`` - which is a required string - and ``"password"`` - which is + a required string which should be encrypted at storage time) and custom + ``injectors`` which define how the inputs are applied to the environment when + a job is run (for example, the value for ``"username"`` should be injected + into an environment variable named ``"MY_USERNAME"``). + + By utilizing these custom ``Credential Types``, customers have the ability to + define custom "Cloud" and "Network" ``Credential Types`` which + modify environment variables, extra vars, and generate file-based + credentials (such as file-based certificates or .ini files) at + `ansible-playbook` runtime. + +* Multiple ``Credentials`` can now be assigned to a ``Job Template`` as long as + the ``Credential Types`` are unique. For example, you can now create a ``Job + Template`` that uses one SSH, one Vault, one EC2, and one Google Compute + Engine credential. You cannot, however, create a ``Job Template`` that uses + two OpenStack credentials. + +* Custom inventory sources can now utilize a ``Credential``; you + can store third-party credentials encrypted within Tower and use their + values from within your custom inventory script (by - for example - reading + an environment variable or a file's contents). + +API Interaction for Credential Management +----------------------------------------- + +``HTTP GET /api/v2/credential_types`` provides a listing of all supported +``Credential Types``, including several read-only types that Tower provides +support for out of the box (SSH, Vault, SCM, Network, Amazon Web Services, +etc...) + +Superusers have the ability to extend Tower by creating, updating, and deleting +new "custom" ``Credential Types``: + + HTTP POST /api/v2/credential_types/ + + { + "name": "Third Party Cloud", + "description": "Integration with Third Party Cloud", + "kind": "cloud", + "inputs": { + "fields": [{ + "id": "api_token", + "label": "API Token", + "type": "string", + "secret": True, + }] + }, + "injectors": { + "env": { + "THIRD_PARTY_CLOUD_API_TOKEN": "{{api_token}}" + } + } + } + +In Tower 3.2, when users create or modify ``Credentials``, they specify the +``credential_type``, and the inputs they pass in are dictated by the +defined ``inputs`` for that ``Credential Type``: + + HTTP POST /api/v2/credentials/ + + { + "name": "Joe's Third Party Cloud API Token", + "description": "", + "organization": , + "user": null, + "team": null, + "credential_type": , + "inputs": { + "api_token": "f239248b-97d0-431b-ae2f-091d80c3452e" + } + } + + HTTP GET /api/v2/credentials/N + + { + "name": "Joe's Third Party Cloud API Token", + "description": "", + "organization": , + "user": null, + "team": null, + "credential_type": , + "inputs": { + "api_token": "$encrypted$" + } + } + +Defining Custom Credential Type Inputs +-------------------------------------- + +A ``Credential Type`` specifies an ``inputs`` schema which defines a set of +ordered fields for that type: + + "inputs": { + "fields": [{ + "id": "api_token", # required - a unique name used to + # reference the field value + + "label": "API Token", # required - a unique label for the + # field + + "help_text": "User-facing short text describing the field.", + + "type": ("string" | "number" | "ssh_private_key") # required, + + "secret": true, # if true, the field will be treated + # as sensitive and stored encrypted + + "multiline": false # if true, the field should be rendered + # as multi-line for input entry + },{ + # field 2... + },{ + # field 3... + }] + "required": ["api_token"] # optional; one or more fields can be marked as required + }, + +As an alternative to static types, fields can also specify multiple choice +strings: + + "inputs": { + "fields": [{ + "id": "api_token", # required - a unique name used to reference the field value + "label": "API Token", # required - a unique label for the field + "choices": ["A", "B", "C"] + }] + }, + +Defining Custom Credential Type Injectors +----------------------------------------- +A ``Credential Type`` can inject ``Credential`` values through the use +of the Jinja templating language (which should be familiar to users of Ansible): + + "injectors": { + "env": { + "THIRD_PARTY_CLOUD_API_TOKEN": "{{api_token}}" + }, + "extra_vars": { + "some_extra_var": "{{username}}:{{password}" + } + } + +``Credential Types`` can also generate temporary files to support .ini files or +certificate/key data: + + "injectors": { + "file": { + "template": "[mycloud]\ntoken={{api_token}}" + }, + "env": { + "MY_CLOUD_INI_FILE": "{{tower.filename}" + } + } + + +Job and Job Template Credential Assignment +------------------------------------------ + +In Tower 3.2, ``Jobs`` and ``Job Templates`` have a new many-to-many +relationship with ``Credential`` that allows selection of multiple +network/cloud credentials. As such, the ``Job`` and ``JobTemplate`` +API resources in `/api/v2/` now have two credential related fields: + + HTTP GET /api/v2/job_templates/N/ + + { + ... + 'credential': + 'vault_credential': + ... + } + +...and a new endpoint for fetching all "extra" credentials: + + HTTP GET /api/v2/job_templates/N/extra_credentials/ + + { + 'count': N, + 'results': [{ + 'name': 'My Credential', + 'credential_type': , + 'inputs': {...}, + ... + }] + } + +Similar to other list attachment/detachment API views, cloud and network +credentials can be created and attached via an `HTTP POST` at this new +endpoint: + + HTTP POST /api/v2/job_templates/N/extra_credentials/ + + { + 'id': , + 'associate': True, + } + + HTTP POST /api/v2/job_templates/N/extra_credentials/ + + { + 'id': , + 'disassociate': True, + } + + HTTP POST /api/v2/job_templates/N/extra_credentials/ + + { + 'name': 'My Credential', + 'credential_type': , + 'inputs': {...}, + ... + } + + +API Backwards Compatability +--------------------------- + +`/api/v1/credentials/` still exists in Tower 3.2, and it transparently works as +before with minimal surprises by attempting to translate `/api/v1/` requests to +the new ``Credential`` and ``Credential Type`` models. + +* When creating or modifying a ``Job Template`` through `v1` of the API, + old-style credential assignment will transparently map to the new model. For + example, the following `POST`'ed payload: + + { + credential: , + vault_credential: , + cloud_credential: , + network_credential: , + } + + ...would transparently update ``JobTemplate.extra_credentials`` to a list + containing both the cloud and network ``Credentials``. + + Similarly, an `HTTP GET /api/v1/job_credentials/N/` will populate + `cloud_credential`, and `network_credential` with the *most recently applied* + matching credential in the list. + +* Custom ``Credentials`` will not be returned in the ``v1`` API; if a user + defines their own ``Credential Type``, its credentials won't show up in the + ``v1`` API. + +* ``HTTP POST`` requests to ``/api/v1/credentials/`` will transparently map + old-style attributes (i.e., ``username``, ``password``, ``ssh_key_data``) to + the appropriate new-style model. Similarly, ``HTTP GET + /api/v1/credentials/N/`` requests will continue to contain old-style + key-value mappings in their payloads. + +* Vault credentials are a new first-level type of credential in Tower 3.2. + As such, any ``Credentials`` pre-Tower 3.2 that contain *both* SSH and Vault + parameters will be migrated to separate distinct ``Credentials`` + post-migration. + + For example, if your Tower 3.1 installation has one ``Credential`` with + a defined ``username``, ``password``, and ``vault_password``, after migration + *two* ``Credentials`` will exist (one which contains the ``username`` and + ``password``, and another which contains only the ``vault_password``). + + +Additional Criteria +------------------- +* Rackspace is being removed from official support in Tower 3.2. Pre-existing + Rackspace Cloud credentials should be automatically migrated to "custom" + credentials. If a customer has never created or used Rackspace Cloud + credentials, the only change they should notice in Tower 3.2 is that + Rackspace is no longer an option provided by Tower when creating/modifying + a Credential. + + +Acceptance Criteria +------------------- +When verifying acceptance we should ensure the following statements are true: + +* `Credential` injection for playbook runs, SCM updates, inventory updates, and + ad-hoc runs should continue to function as they did prior to Tower 3.2 for the + `Credential Types` provided by Tower. +* It should be possible to create and modify every type of `Credential` supported + prior to Tower 3.2 (SSH, SCM, EC2, etc..., with the exception of Rackspace). +* Superusers (and only superusers) should be able to define custom `Credential + Types`. They should properly inject environment variables, extra vars, and + files for playbook runs, SCM updates, inventory updates, and ad-hoc runs. +* The default `Credential Types` included with Tower in 3.2 should be + non-editable/readonly and cannot be deleted by any user. +* Stored `Credential` values for _all_ types should be consistent before and + after Tower 3.2 migration/upgrade. +* `Job Templates` should be able to specify multiple extra `Credentials` as + defined in the constraints in this document. +* Custom inventory sources should be able to specify a cloud/network + `Credential` and they should properly update the environment (environment + variables, extra vars, written files) when an inventory source update runs. +* If a `Credential Type` is being used by one or more `Credentials`, the fields + defined in its ``inputs`` should be read-only. +* `Credential Types` should support activity stream history for basic object + modification.