Introduce a new CredentialTemplate model

Credentials now have a required CredentialType, which defines inputs
(i.e., username, password) and injectors (i.e., assign the username to
SOME_ENV_VARIABLE at job runtime)

This commit only implements the model changes necessary to support the
new inputs model, and includes code for the credential serializer that
allows backwards-compatible support for /api/v1/credentials/; tasks.py
still needs to be updated to actually respect CredentialType injectors.

This change *will* break the UI for credentials (because it needs to be
updated to use the new v2 endpoint).

see: #5877
see: #5876
see: #5805

awx/api/fields.py

@@ -1,13 +1,11 @@
 # Copyright (c) 2016 Ansible, Inc.
 # All Rights Reserved.
 
-# Django
-from django.utils.encoding import force_text
 
 # Django REST Framework
 from rest_framework import serializers
 
-__all__ = ['BooleanNullField', 'CharNullField', 'ChoiceNullField', 'EncryptedPasswordField', 'VerbatimField']
+__all__ = ['BooleanNullField', 'CharNullField', 'ChoiceNullField', 'VerbatimField']
 
 
 class NullFieldMixin(object):
@@ -58,25 +56,6 @@ class ChoiceNullField(NullFieldMixin, serializers.ChoiceField):
         return super(ChoiceNullField, self).to_internal_value(data or u'')
 
 
-class EncryptedPasswordField(CharNullField):
-    '''
-    Custom field to handle encrypted password values (on credentials).
-    '''
-
-    def to_internal_value(self, data):
-        value = super(EncryptedPasswordField, self).to_internal_value(data or u'')
-        # If user submits a value starting with $encrypted$, ignore it.
-        if force_text(value).startswith('$encrypted$'):
-            raise serializers.SkipField
-        return value
-
-    def to_representation(self, value):
-        # Replace the actual encrypted value with the string $encrypted$.
-        if force_text(value).startswith('$encrypted$'):
-            return '$encrypted$'
-        return value
-
-
 class VerbatimField(serializers.Field):
     '''
     Custom field that passes the value through without changes.

awx/api/filters.py

@@ -6,7 +6,7 @@ import re
 import json
 
 # Django
-from django.core.exceptions import FieldError, ValidationError
+from django.core.exceptions import FieldError, ValidationError, ObjectDoesNotExist
 from django.db import models
 from django.db.models import Q
 from django.db.models.fields import FieldDoesNotExist
@@ -22,6 +22,7 @@ from rest_framework.filters import BaseFilterBackend
 
 # Ansible Tower
 from awx.main.utils import get_type_for_model, to_python_boolean
+from awx.main.models.credential import CredentialType
 from awx.main.models.rbac import RoleAncestorEntry
 
 
@@ -161,6 +162,18 @@ class FieldLookupBackend(BaseFilterBackend):
         except UnicodeEncodeError:
             raise ValueError("%r is not an allowed field name. Must be ascii encodable." % lookup)
 
+        # Make legacy v1 Credential fields work for backwards compatability
+        # TODO: remove after API v1 deprecation period
+        if model._meta.object_name == 'Credential' and lookup == 'kind':
+            try:
+                type_ = CredentialType.from_v1_kind(value)
+                if type_ is None:
+                    raise ParseError(_('cannot filter on kind %s') % value)
+                value = type_.pk
+                lookup = 'credential_type'
+            except ObjectDoesNotExist as e:
+                raise ParseError(_('cannot filter on kind %s') % value)
+
         field, new_lookup = self.get_field_from_lookup(model, lookup)
 
         # Type names are stored without underscores internally, but are presented and

awx/api/serializers.py

@@ -48,8 +48,8 @@ from awx.main.utils import (
 from awx.main.validators import vars_validate_or_raise
 
 from awx.conf.license import feature_enabled
-from awx.api.versioning import reverse
-from awx.api.fields import BooleanNullField, CharNullField, ChoiceNullField, EncryptedPasswordField, VerbatimField
+from awx.api.versioning import reverse, get_request_version
+from awx.api.fields import BooleanNullField, CharNullField, ChoiceNullField, VerbatimField
 
 logger = logging.getLogger('awx.api.serializers')
 
@@ -243,6 +243,12 @@ class BaseSerializer(serializers.ModelSerializer):
     created       = serializers.SerializerMethodField()
     modified      = serializers.SerializerMethodField()
 
+    @property
+    def version(self):
+        """
+        The request version component of the URL as an integer i.e., 1 or 2
+        """
+        return get_request_version(self.context.get('request'))
 
     def get_type(self, obj):
         return get_type_for_model(self.Meta.model)
@@ -309,7 +315,18 @@ class BaseSerializer(serializers.ModelSerializer):
                     continue
                 summary_fields[fk] = OrderedDict()
                 for field in related_fields:
+
                     fval = getattr(fkval, field, None)
+
+                    # TODO: remove when API v1 is removed
+                    if all([
+                        self.version == 1,
+                        'credential' in fk,
+                        field == 'kind',
+                        fval == 'machine'
+                    ]):
+                        fval = 'ssh'
+
                     if fval is None and field == 'type':
                         if isinstance(fkval, PolymorphicModel):
                             fkval = fkval.get_real_instance()
@@ -1819,25 +1836,76 @@ class ResourceAccessListElementSerializer(UserSerializer):
         return ret
 
 
-class CredentialSerializer(BaseSerializer):
+class CredentialTypeSerializer(BaseSerializer):
     show_capabilities = ['edit', 'delete']
 
+    class Meta:
+        model = CredentialType
+        fields = ('*', 'kind', 'name', 'managed_by_tower', 'inputs',
+                  'injectors')
+
+
+# TODO: remove when API v1 is removed
+@six.add_metaclass(BaseSerializerMetaclass)
+class V1CredentialFields(BaseSerializer):
+
     class Meta:
         model = Credential
         fields = ('*', 'kind', 'cloud', 'host', 'username',
                   'password', 'security_token', 'project', 'domain',
-                  'ssh_key_data', 'ssh_key_unlock', 'organization',
-                  'become_method', 'become_username', 'become_password',
-                  'vault_password', 'subscription', 'tenant', 'secret', 'client',
-                  'authorize', 'authorize_password')
+                  'ssh_key_data', 'ssh_key_unlock', 'become_method',
+                  'become_username', 'become_password', 'vault_password',
+                  'subscription', 'tenant', 'secret', 'client', 'authorize',
+                  'authorize_password')
 
-    def build_standard_field(self, field_name, model_field):
-        field_class, field_kwargs = super(CredentialSerializer, self).build_standard_field(field_name, model_field)
-        if field_name in Credential.PASSWORD_FIELDS:
-            field_class = EncryptedPasswordField
-            field_kwargs['required'] = False
-            field_kwargs['default'] = ''
-        return field_class, field_kwargs
+    def build_field(self, field_name, info, model_class, nested_depth):
+        if field_name in V1Credential.FIELDS:
+            return self.build_standard_field(field_name,
+                                             V1Credential.FIELDS[field_name])
+        return super(V1CredentialFields, self).build_field(field_name, info, model_class, nested_depth)
+
+
+@six.add_metaclass(BaseSerializerMetaclass)
+class V2CredentialFields(BaseSerializer):
+
+    class Meta:
+        model = Credential
+        fields = ('*', 'credential_type', 'inputs')
+
+
+class CredentialSerializer(BaseSerializer):
+    show_capabilities = ['edit', 'delete']
+
+    class Meta:
+        model = Credential
+        fields = ('*', 'organization')
+
+    def get_fields(self):
+        fields = super(CredentialSerializer, self).get_fields()
+
+        # TODO: remove when API v1 is removed
+        if self.version == 1:
+            fields.update(V1CredentialFields().get_fields())
+        else:
+            fields.update(V2CredentialFields().get_fields())
+        return fields
+
+    def to_representation(self, data):
+        value = super(CredentialSerializer, self).to_representation(data)
+
+        # TODO: remove when API v1 is removed
+        if self.version == 1:
+            if value.get('kind') == 'machine':
+                value['kind'] = 'ssh'
+
+            for field in V1Credential.PASSWORD_FIELDS:
+                if field in value and force_text(value[field]).startswith('$encrypted$'):
+                    value[field] = '$encrypted$'
+
+        for k, v in value.get('inputs', {}).items():
+            if force_text(v).startswith('$encrypted$'):
+                value['inputs'][k] = '$encrypted$'
+        return value
 
     def get_related(self, obj):
         res = super(CredentialSerializer, self).get_related(obj)
@@ -1853,6 +1921,12 @@ class CredentialSerializer(BaseSerializer):
             owner_teams = self.reverse('api:credential_owner_teams_list', kwargs={'pk': obj.pk}),
         ))
 
+        # TODO: remove when API v1 is removed
+        if self.version > 1:
+            res.update(dict(
+                credential_type = self.reverse('api:credential_type_detail', kwargs={'pk': obj.credential_type.pk}),
+            ))
+
         parents = [role for role in obj.admin_role.parents.all() if role.object_id is not None]
         if parents:
             res.update({parents[0].content_type.name:parents[0].content_object.get_absolute_url(self.context.get('request'))})
@@ -1886,6 +1960,35 @@ class CredentialSerializer(BaseSerializer):
 
         return summary_dict
 
+    def get_validation_exclusions(self, obj=None):
+        # CredentialType is now part of validation; legacy v1 fields (e.g.,
+        # 'username', 'password') in JSON POST payloads use the
+        # CredentialType's inputs definition to determine their validity
+        ret = super(CredentialSerializer, self).get_validation_exclusions(obj)
+        for field in ('credential_type', 'inputs'):
+            if field in ret:
+                ret.remove(field)
+        return ret
+
+    def to_internal_value(self, data):
+        if 'credential_type' not in data:
+            # If `credential_type` is not provided, assume the payload is a
+            # v1 credential payload that specifies a `kind` and a flat list
+            # of field values
+            #
+            # In this scenario, we should automatically detect the proper
+            # CredentialType based on the provided values
+            kind = data.get('kind', 'ssh')
+            credential_type = CredentialType.from_v1_kind(kind, data)
+            data['credential_type'] = credential_type.pk
+            value = OrderedDict(
+                {'credential_type': credential_type}.items() +
+                super(CredentialSerializer, self).to_internal_value(data).items()
+            )
+            value.pop('kind', None)
+            return value
+        return super(CredentialSerializer, self).to_internal_value(data)
+
 
 class CredentialSerializerCreate(CredentialSerializer):
 
@@ -1926,7 +2029,20 @@ class CredentialSerializerCreate(CredentialSerializer):
         team = validated_data.pop('team', None)
         if team:
             validated_data['organization'] = team.organization
+
+        # If our payload contains v1 credential fields, translate to the new
+        # model
+        # TODO: remove when API v1 is removed
+        if self.version == 1:
+            for attr in (
+                set(V1Credential.FIELDS) & set(validated_data.keys())  # set intersection
+            ):
+                validated_data.setdefault('inputs', {})
+                value = validated_data.pop(attr)
+                if value:
+                    validated_data['inputs'][attr] = value
         credential = super(CredentialSerializerCreate, self).create(validated_data)
+
         if user:
             credential.admin_role.members.add(user)
         if team:

awx/api/urls.py

@@ -164,6 +164,11 @@ inventory_script_urls = patterns('awx.api.views',
     url(r'^(?P<pk>[0-9]+)/object_roles/$',              'inventory_script_object_roles_list'),
 )
 
+credential_type_urls = patterns('awx.api.views',
+    url(r'^$',                                          'credential_type_list'),
+    url(r'^(?P<pk>[0-9]+)/$',                           'credential_type_detail'),
+)
+
 credential_urls = patterns('awx.api.views',
     url(r'^$',                                          'credential_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$',           'credential_activity_stream_list'),
@@ -378,7 +383,13 @@ v1_urls = patterns('awx.api.views',
     url(r'^activity_stream/',       include(activity_stream_urls)),
 )
 
+v2_urls = patterns('awx.api.views',
+    url(r'^$',                      'api_version_root_view'),
+    url(r'^credential_types/',     include(credential_type_urls)),
+)
+
 urlpatterns = patterns('awx.api.views',
     url(r'^$', 'api_root_view'),
+    url(r'^(?P<version>(v2))/', include(v2_urls)),
     url(r'^(?P<version>(v1|v2))/', include(v1_urls))
 )

awx/api/versioning.py

@@ -1,10 +1,22 @@
 # Copyright (c) 2017 Ansible by Red Hat
 # All Rights Reserved.
 
+from django.conf import settings
+
 from rest_framework.reverse import reverse as drf_reverse
 from rest_framework.versioning import URLPathVersioning as BaseVersioning
 
 
+def get_request_version(request):
+    """
+    The API version of a request as an integer i.e., 1 or 2
+    """
+    version = settings.REST_FRAMEWORK['DEFAULT_VERSION']
+    if request and hasattr(request, 'version'):
+        version = request.version
+    return int(version.lstrip('v'))
+
+
 def reverse(viewname, args=None, kwargs=None, request=None, format=None, **extra):
     if request is None or getattr(request, 'version', None) is None:
         # We need the "current request" to determine the correct version to
@@ -13,7 +25,7 @@ def reverse(viewname, args=None, kwargs=None, request=None, format=None, **extra
         if kwargs is None:
             kwargs = {}
         if 'version' not in kwargs:
-            kwargs['version'] = 'v2'
+            kwargs['version'] = settings.REST_FRAMEWORK['DEFAULT_VERSION']
     return drf_reverse(viewname, args, kwargs, request, format, **extra)
 
 

awx/api/views.py

@@ -64,7 +64,7 @@ from awx.main.ha import is_ha_environment
 from awx.api.authentication import TaskAuthentication, TokenGetAuthentication
 from awx.api.generics import get_view_name
 from awx.api.generics import * # noqa
-from awx.api.versioning import reverse
+from awx.api.versioning import reverse, get_request_version
 from awx.conf.license import get_license, feature_enabled, feature_exists, LicenseForbids
 from awx.main.models import * # noqa
 from awx.main.utils import * # noqa
@@ -155,7 +155,6 @@ class ApiVersionRootView(APIView):
 
     def get(self, request, format=None):
         ''' list top level resources '''
-
         data = OrderedDict()
         data['authtoken'] = reverse('api:auth_token_view', request=request)
         data['ping'] = reverse('api:api_v1_ping_view', request=request)
@@ -169,6 +168,8 @@ class ApiVersionRootView(APIView):
         data['project_updates'] = reverse('api:project_update_list', request=request)
         data['teams'] = reverse('api:team_list', request=request)
         data['credentials'] = reverse('api:credential_list', request=request)
+        if get_request_version(request) > 1:
+            data['credential_types'] = reverse('api:credential_type_list', request=request)
         data['inventory'] = reverse('api:inventory_list', request=request)
         data['inventory_scripts'] = reverse('api:inventory_script_list', request=request)
         data['inventory_sources'] = reverse('api:inventory_source_list', request=request)
@@ -1476,6 +1477,20 @@ class UserAccessList(ResourceAccessList):
     new_in_300 = True
 
 
+class CredentialTypeList(ListCreateAPIView):
+
+    model = CredentialType
+    serializer_class = CredentialTypeSerializer
+    new_in_320 = True
+
+
+class CredentialTypeDetail(RetrieveUpdateDestroyAPIView):
+
+    model = CredentialType
+    serializer_class = CredentialTypeSerializer
+    new_in_320 = True
+
+
 class CredentialList(ListCreateAPIView):
 
     model = Credential