define native CredentialType inputs/injectors in code, not in the DB

This has a few benefits:

1.  It makes adding new fields to built-in CredentialTypes _much_
    simpler.  In the past, we've had to write a migration every time we
    want to modify an existing type (changing a label/help text,
    changing options like the recent become_method changes) or
    when adding a new field entirely

2.  It paves the way for third party credential plugins support, where
    importable libraries will define their own source code-based schema

awx/main/migrations/0061_v350_track_native_credentialtype_source.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.16 on 2019-02-19 04:27
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+from awx.main.models import CredentialType
+
+
+def migrate_to_static_inputs(apps, schema_editor):
+    CredentialType.setup_tower_managed_defaults()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0060_v350_update_schedule_uniqueness_constraint'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='credentialtype',
+            name='namespace',
+            field=models.CharField(default=None, editable=False, max_length=1024, null=True),
+        ),
+        migrations.RunPython(migrate_to_static_inputs)
+    ]

awx/main/migrations/_credentialtypes.py

@@ -62,155 +62,40 @@ def _disassociate_non_insights_projects(apps, cred):
 
 
 def migrate_to_v2_credentials(apps, schema_editor):
-    CredentialType.setup_tower_managed_defaults()
+    # TODO: remove once legacy/EOL'd Towers no longer support this upgrade path
-    deprecated_cred = _generate_deprecated_cred_types()
+    pass
-
-    # this monkey-patch is necessary to make the implicit role generation save
-    # signal use the correct Role model (the version active at this point in
-    # migration, not the one at HEAD)
-    orig_current_apps = utils.get_current_apps
-    try:
-        utils.get_current_apps = lambda: apps
-        for cred in apps.get_model('main', 'Credential').objects.all():
-            job_templates = cred.jobtemplates.all()
-            jobs = cred.jobs.all()
-            data = {}
-            if getattr(cred, 'vault_password', None):
-                data['vault_password'] = cred.vault_password
-            if _is_insights_scm(apps, cred):
-                _disassociate_non_insights_projects(apps, cred)
-                credential_type = _get_insights_credential_type()
-            else:
-                credential_type = _populate_deprecated_cred_types(deprecated_cred, cred.kind) or CredentialType.from_v1_kind(cred.kind, data)
-
-            defined_fields = credential_type.defined_fields
-            cred.credential_type = apps.get_model('main', 'CredentialType').objects.get(pk=credential_type.pk)
-
-            for field in defined_fields:
-                if getattr(cred, field, None):
-                    cred.inputs[field] = getattr(cred, field)
-            if cred.vault_password:
-                for jt in job_templates:
-                    jt.credential = None
-                    jt.vault_credential = cred
-                    jt.save()
-                for job in jobs:
-                    job.credential = None
-                    job.vault_credential = cred
-                    job.save()
-            if data.get('is_insights', False):
-                cred.kind = 'insights'
-            cred.save()
-
-            #
-            # If the credential contains a vault password, create a new
-            # *additional* credential for the ssh details
-            #
-            if cred.vault_password:
-                # We need to make an ssh credential, too
-                ssh_type = CredentialType.from_v1_kind('ssh')
-                new_cred = apps.get_model('main', 'Credential').objects.get(pk=cred.pk)
-                new_cred.pk = None
-                new_cred.vault_password = ''
-                new_cred.credential_type = apps.get_model('main', 'CredentialType').objects.get(pk=ssh_type.pk)
-                if 'vault_password' in new_cred.inputs:
-                    del new_cred.inputs['vault_password']
-
-                # unset these attributes so that new roles are properly created
-                # at save time
-                new_cred.read_role = None
-                new_cred.admin_role = None
-                new_cred.use_role = None
-
-                if any([getattr(cred, field) for field in ssh_type.defined_fields]):
-                    new_cred.save(force_insert=True)
-
-                    # copy rbac roles
-                    for role_type in ('read_role', 'admin_role', 'use_role'):
-                        for member in getattr(cred, role_type).members.all():
-                            getattr(new_cred, role_type).members.add(member)
-                        for role in getattr(cred, role_type).parents.all():
-                            getattr(new_cred, role_type).parents.add(role)
-
-                    for jt in job_templates:
-                        jt.credential = new_cred
-                        jt.save()
-                    for job in jobs:
-                        job.credential = new_cred
-                        job.save()
-
-                    # passwords must be decrypted and re-encrypted, because
-                    # their encryption is based on the Credential's primary key
-                    # (which has changed)
-                    for field in ssh_type.defined_fields:
-                        if field in ssh_type.secret_fields:
-                            value = decrypt_field(cred, field)
-                            if value:
-                                setattr(new_cred, field, value)
-                                new_cred.inputs[field] = encrypt_field(new_cred, field)
-                                setattr(new_cred, field, '')
-                        elif getattr(cred, field):
-                            new_cred.inputs[field] = getattr(cred, field)
-                    new_cred.save()
-    finally:
-        utils.get_current_apps = orig_current_apps
 
 
 def migrate_job_credentials(apps, schema_editor):
-    # this monkey-patch is necessary to make the implicit role generation save
+    # TODO: remove once legacy/EOL'd Towers no longer support this upgrade path
-    # signal use the correct Role model (the version active at this point in
+    pass
-    # migration, not the one at HEAD)
-    orig_current_apps = utils.get_current_apps
-    try:
-        utils.get_current_apps = lambda: apps
-        for type_ in ('Job', 'JobTemplate'):
-            for obj in apps.get_model('main', type_).objects.all():
-                if obj.cloud_credential:
-                    obj.extra_credentials.add(obj.cloud_credential)
-                if obj.network_credential:
-                    obj.extra_credentials.add(obj.network_credential)
-                obj.save()
-    finally:
-        utils.get_current_apps = orig_current_apps
 
 
 def add_vault_id_field(apps, schema_editor):
-    vault_credtype = CredentialType.objects.get(kind='vault')
+    # this is no longer necessary; schemas are defined in code
-    vault_credtype.inputs = CredentialType.defaults.get('vault')().inputs
+    pass
-    vault_credtype.save()
 
 
 def remove_vault_id_field(apps, schema_editor):
-    vault_credtype = CredentialType.objects.get(kind='vault')
+    # this is no longer necessary; schemas are defined in code
-    idx = 0
+    pass
-    for i, input in enumerate(vault_credtype.inputs['fields']):
-        if input['id'] == 'vault_id':
-            idx = i
-            break
-    vault_credtype.inputs['fields'].pop(idx)
-    vault_credtype.save()
 
 
 def create_rhv_tower_credtype(apps, schema_editor):
-    CredentialType.setup_tower_managed_defaults()
+    # this is no longer necessary; schemas are defined in code
+    pass
 
 
 def add_tower_verify_field(apps, schema_editor):
-    tower_credtype = CredentialType.objects.get(
+    # this is no longer necessary; schemas are defined in code
-        kind='cloud', name='Ansible Tower', managed_by_tower=True
+    pass
-    )
-    tower_credtype.inputs = CredentialType.defaults.get('tower')().inputs
-    tower_credtype.save()
 
 
 def add_azure_cloud_environment_field(apps, schema_editor):
-    azure_rm_credtype = CredentialType.objects.get(kind='cloud',
+    # this is no longer necessary; schemas are defined in code
-                                                   name='Microsoft Azure Resource Manager')
+    pass
-    azure_rm_credtype.inputs = CredentialType.defaults.get('azure_rm')().inputs
-    azure_rm_credtype.save()
 
 
 def remove_become_methods(apps, schema_editor):
-    become_credtype = CredentialType.objects.filter(kind='ssh', managed_by_tower=True).first()
+    # this is no longer necessary; schemas are defined in code
-    become_credtype.inputs = CredentialType.defaults.get('ssh')().inputs
+    pass
-    become_credtype.save()

awx/main/models/__init__.py

@@ -16,7 +16,7 @@ from awx.main.models.organization import (  # noqa
     Organization, Profile, Team, UserSessionMembership
 )
 from awx.main.models.credential import (  # noqa
-    Credential, CredentialType, V1Credential, build_safe_env
+    Credential, CredentialType, ManagedCredentialType, V1Credential, build_safe_env
 )
 from awx.main.models.projects import Project, ProjectUpdate  # noqa
 from awx.main.models.inventory import (  # noqa

awx/main/models/credential/__init__.py

@@ -1,12 +1,13 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
-from collections import OrderedDict
 import functools
+import inspect
 import logging
 import os
 import re
 import stat
 import tempfile
+from types import SimpleNamespace
 
 # Jinja2
 from jinja2 import Template
@@ -22,7 +23,7 @@ from awx.api.versioning import reverse
 from awx.main.fields import (ImplicitRoleField, CredentialInputField,
                              CredentialTypeInputField,
                              CredentialTypeInjectorField)
-from awx.main.utils import decrypt_field
+from awx.main.utils import decrypt_field, classproperty
 from awx.main.utils.safe_yaml import safe_dump
 from awx.main.validators import validate_ssh_private_key
 from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel
@@ -465,8 +466,6 @@ class CredentialType(CommonModelNameNotUnique):
     output injectors (i.e., an environment variable that uses the API key).
     '''
 
-    defaults = OrderedDict()
-
     class Meta:
         app_label = 'main'
         ordering = ('kind', 'name')
@@ -489,6 +488,12 @@ class CredentialType(CommonModelNameNotUnique):
         default=False,
         editable=False
     )
+    namespace = models.CharField(
+        max_length=1024,
+        null=True,
+        default=None,
+        editable=False
+    )
     inputs = CredentialTypeInputField(
         blank=True,
         default={},
@@ -504,6 +509,15 @@ class CredentialType(CommonModelNameNotUnique):
                     'Ansible Tower documentation for example syntax.')
     )
 
+    @classmethod
+    def from_db(cls, db, field_names, values):
+        instance = super(CredentialType, cls).from_db(db, field_names, values)
+        if instance.managed_by_tower and instance.namespace:
+            native = ManagedCredentialType.registry[instance.namespace]
+            instance.inputs = native.inputs
+            instance.injectors = native.injectors
+        return instance
+
     def get_absolute_url(self, request=None):
         # Page does not exist in API v1
         if request.version == 'v1':
@@ -539,23 +553,29 @@ class CredentialType(CommonModelNameNotUnique):
                     return field['choices'][0]
                 return {'string': '', 'boolean': False}[field['type']]
 
-    @classmethod
+    @classproperty
-    def default(cls, f):
+    def defaults(cls):
-        func = functools.partial(f, cls)
+        return dict(
-        cls.defaults[f.__name__] = func
+            (k, functools.partial(v.create))
-        return func
+            for k, v in ManagedCredentialType.registry.items()
+        )
 
     @classmethod
-    def setup_tower_managed_defaults(cls, persisted=True):
+    def setup_tower_managed_defaults(cls):
-        for default in cls.defaults.values():
+        for default in ManagedCredentialType.registry.values():
-            default_ = default()
+            existing = CredentialType.objects.filter(name=default.name, kind=default.kind).first()
-            if persisted:
+            if existing is not None:
-                if CredentialType.objects.filter(name=default_.name, kind=default_.kind).count():
+                existing.namespace = default.namespace
+                existing.inputs = {}
+                existing.injectors = {}
+                existing.save()
                 continue
             logger.debug(_(
-                    "adding %s credential type" % default_.name
+                "adding %s credential type" % default.name
             ))
-                default_.save()
+            created = default.create()
+            created.inputs = created.injectors = {}
+            created.save()
 
     @classmethod
     def from_v1_kind(cls, kind, data={}):
@@ -701,12 +721,39 @@ class CredentialType(CommonModelNameNotUnique):
                 safe_args.extend(['-e', '@%s' % path])
 
 
-@CredentialType.default
+class ManagedCredentialType(SimpleNamespace):
-def ssh(cls):
+
-    return cls(
+    registry = {}
+
+    def __init__(self, namespace, **kwargs):
+        for k in ('inputs', 'injectors'):
+            if k not in kwargs:
+                kwargs[k] = {}
+        super(ManagedCredentialType, self).__init__(namespace=namespace, **kwargs)
+        if namespace in ManagedCredentialType.registry:
+            raise ValueError(
+                'a ManagedCredentialType with namespace={} is already defined in {}'.format(
+                    namespace,
+                    inspect.getsourcefile(ManagedCredentialType.registry[namespace].__class__)
+                )
+            )
+        ManagedCredentialType.registry[namespace] = self
+
+    def create(self):
+        return CredentialType(
+            namespace=self.namespace,
+            kind=self.kind,
+            name=self.name,
+            managed_by_tower=True,
+            inputs=self.inputs,
+            injectors=self.injectors,
+        )
+
+
+ManagedCredentialType(
+    namespace='ssh',
     kind='ssh',
     name=ugettext_noop('Machine'),
-        managed_by_tower=True,
     inputs={
         'fields': [{
             'id': 'username',
@@ -755,10 +802,8 @@ def ssh(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='scm',
-def scm(cls):
-    return cls(
     kind='scm',
     name=ugettext_noop('Source Control'),
     managed_by_tower=True,
@@ -791,10 +836,8 @@ def scm(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='vault',
-def vault(cls):
-    return cls(
     kind='vault',
     name=ugettext_noop('Vault'),
     managed_by_tower=True,
@@ -820,10 +863,8 @@ def vault(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='net',
-def net(cls):
-    return cls(
     kind='net',
     name=ugettext_noop('Network'),
     managed_by_tower=True,
@@ -867,10 +908,8 @@ def net(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='aws',
-def aws(cls):
-    return cls(
     kind='cloud',
     name=ugettext_noop('Amazon Web Services'),
     managed_by_tower=True,
@@ -898,10 +937,8 @@ def aws(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='openstack',
-def openstack(cls):
-    return cls(
     kind='cloud',
     name=ugettext_noop('OpenStack'),
     managed_by_tower=True,
@@ -938,10 +975,8 @@ def openstack(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='vmware',
-def vmware(cls):
-    return cls(
     kind='cloud',
     name=ugettext_noop('VMware vCenter'),
     managed_by_tower=True,
@@ -966,10 +1001,8 @@ def vmware(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='satellite6',
-def satellite6(cls):
-    return cls(
     kind='cloud',
     name=ugettext_noop('Red Hat Satellite 6'),
     managed_by_tower=True,
@@ -994,10 +1027,8 @@ def satellite6(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='cloudforms',
-def cloudforms(cls):
-    return cls(
     kind='cloud',
     name=ugettext_noop('Red Hat CloudForms'),
     managed_by_tower=True,
@@ -1023,10 +1054,8 @@ def cloudforms(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='gce',
-def gce(cls):
-    return cls(
     kind='cloud',
     name=ugettext_noop('Google Compute Engine'),
     managed_by_tower=True,
@@ -1059,10 +1088,8 @@ def gce(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='azure_rm',
-def azure_rm(cls):
-    return cls(
     kind='cloud',
     name=ugettext_noop('Microsoft Azure Resource Manager'),
     managed_by_tower=True,
@@ -1106,10 +1133,8 @@ def azure_rm(cls):
     }
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='insights',
-def insights(cls):
-    return cls(
     kind='insights',
     name=ugettext_noop('Insights'),
     managed_by_tower=True,
@@ -1134,10 +1159,8 @@ def insights(cls):
     },
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='rhv',
-def rhv(cls):
-    return cls(
     kind='cloud',
     name=ugettext_noop('Red Hat Virtualization'),
     managed_by_tower=True,
@@ -1186,10 +1209,8 @@ def rhv(cls):
     },
 )
 
-
+ManagedCredentialType(
-@CredentialType.default
+    namespace='tower',
-def tower(cls):
-    return cls(
     kind='cloud',
     name=ugettext_noop('Ansible Tower'),
     managed_by_tower=True,

awx/main/utils/common.py

@@ -47,7 +47,7 @@ __all__ = ['get_object_or_400', 'camelcase_to_underscore', 'memoize', 'memoize_d
            'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict',
            'model_instance_diff', 'timestamp_apiformat', 'parse_yaml_or_json', 'RequireDebugTrueOrTest',
            'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', 'get_custom_venv_choices', 'get_external_account',
-           'task_manager_bulk_reschedule', 'schedule_task_manager']
+           'task_manager_bulk_reschedule', 'schedule_task_manager', 'classproperty']
 
 
 def get_object_or_400(klass, *args, **kwargs):
@@ -1113,3 +1113,17 @@ def get_external_account(user):
             getattr(settings, 'TACACSPLUS_HOST', None)) and user.enterprise_auth.all():
         account_type = "enterprise"
     return account_type
+
+
+class classproperty:
+
+    def __init__(self, fget=None, fset=None, fdel=None, doc=None):
+        self.fget = fget
+        self.fset = fset
+        self.fdel = fdel
+        if doc is None and fget is not None:
+            doc = fget.__doc__
+        self.__doc__ = doc
+
+    def __get__(self, instance, ownerclass):
+        return self.fget(ownerclass)