move path parameterization to the CredentialInputSource model

awx/api/serializers.py

@@ -2833,6 +2833,7 @@ class CredentialInputSourceSerializer(BaseSerializer):
             'type',
             'url',
             'input_field_name',
+            'metadata',
             'target_credential',
             'source_credential',
             'source_credential_type',

awx/main/credential_plugins/azure_kv.py

@@ -22,7 +22,8 @@ azure_keyvault_inputs = {
         'id': 'tenant',
         'label': 'Tenant ID',
         'type': 'string'
-    }, {
+    }],
+    'metadata': [{
         'id': 'secret_field',
         'label': 'Secret Name',
         'type': 'string',
@@ -33,7 +34,7 @@ azure_keyvault_inputs = {
         'type': 'string',
         'help_text': 'Used to specify a specific secret version (if left empty, the latest version will be used).',
     }],
-    'required': ['url', 'client', 'secret', 'tenant'],
+    'required': ['url', 'client', 'secret', 'tenant', 'secret_field'],
 }
 
 

awx/main/credential_plugins/hashivault.py

@@ -1,3 +1,4 @@
+import copy
 import os
 import pathlib
 
@@ -18,7 +19,8 @@ base_inputs = {
         'type': 'string',
         'secret': True,
         'help_text': 'The access token used to authenticate to the Vault server',
-    }, {
+    }],
+    'metadata': [{
         'id': 'secret_path',
         'label': 'Path to Secret',
         'type': 'string',
@@ -27,50 +29,49 @@ base_inputs = {
     'required': ['url', 'token', 'secret_path'],
 }
 
-hashi_kv_inputs = {
+hashi_kv_inputs = copy.deepcopy(base_inputs)
-    'fields': base_inputs['fields'] + [{
+hashi_kv_inputs['fields'].append({
-        'id': 'secret_field',
+    'id': 'api_version',
-        'label': 'Key Name',
+    'label': 'API Version',
-        'type': 'string',
+    'choices': ['v1', 'v2'],
-        'help_text': 'The name of the key to look up in the secret.',
+    'help_text': 'API v1 is for static key/value lookups.  API v2 is for versioned key/value lookups.',
-    }, {
+    'default': 'v1',
-        'id': 'secret_version',
+})
-        'label': 'Secret Version (v2 only)',
+hashi_kv_inputs['metadata'].extend([{
-        'type': 'string',
+    'id': 'secret_key',
-        'help_text': 'Used to specify a specific secret version (if left empty, the latest version will be used).',
+    'label': 'Key Name',
-    }, {
+    'type': 'string',
-        'id': 'api_version',
+    'help_text': 'The name of the key to look up in the secret.',
-        'label': 'API Version',
+}, {
-        'choices': ['v1', 'v2'],
+    'id': 'secret_version',
-        'help_text': 'API v1 is for static key/value lookups.  API v2 is for versioned key/value lookups.',
+    'label': 'Secret Version (v2 only)',
-        'default': 'v1',
+    'type': 'string',
-    }],
+    'help_text': 'Used to specify a specific secret version (if left empty, the latest version will be used).',
-    'required': base_inputs['required'] + ['secret_field', 'api_version']
+}])
-}
+hashi_kv_inputs['required'].extend(['api_version', 'secret_key'])
 
-hashi_ssh_inputs = {
+hashi_ssh_inputs = copy.deepcopy(base_inputs)
-    'fields': base_inputs['fields'] + [{
+hashi_ssh_inputs['metadata'].extend([{
-        'id': 'role',
+    'id': 'role',
-        'label': 'Role Name',
+    'label': 'Role Name',
-        'type': 'string',
+    'type': 'string',
-        'help_text': 'The name of the role used to sign.'
+    'help_text': 'The name of the role used to sign.'
-    }, {
+}, {
-        'id': 'valid_principals',
+    'id': 'valid_principals',
-        'label': 'Valid Principals',
+    'label': 'Valid Principals',
-        'type': 'string',
+    'type': 'string',
-        'help_text': 'Valid principals (either usernames or hostnames) that the certificate should be signed for.',
+    'help_text': 'Valid principals (either usernames or hostnames) that the certificate should be signed for.',
-    }],
+}])
-    'required': base_inputs['required'] + ['role']
+hashi_ssh_inputs['required'].extend(['role'])
-}
 
 
 def kv_backend(raw, **kwargs):
     token = kwargs['token']
     url = kwargs['url']
     secret_path = kwargs['secret_path']
-    secret_field = kwargs.get('secret_field', None)
+    secret_key = kwargs.get('secret_key', None)
 
-    api_version = kwargs.get('api_version', None)
+    api_version = kwargs['api_version']
 
     client = Client(url=url, token=token, verify=True)
     if api_version == 'v2':
@@ -99,12 +100,12 @@ def kv_backend(raw, **kwargs):
             'could not read secret {} from {}'.format(secret_path, url)
         )
 
-    if secret_field:
+    if secret_key:
         try:
-            return response['data'][secret_field]
+            return response['data'][secret_key]
         except KeyError:
             raise RuntimeError(
-                '{} is not present at {}'.format(secret_field, secret_path)
+                '{} is not present at {}'.format(secret_key, secret_path)
             )
     return response['data']
 

awx/main/fields.py

@@ -480,6 +480,69 @@ def format_ssh_private_key(value):
     return True
 
 
+class DynamicCredentialInputField(JSONSchemaField):
+    """
+    Used to validate JSON for
+    `awx.main.models.credential:CredentialInputSource().metadata`.
+
+    Metadata for input sources is represented as a dictionary e.g.,
+    {'secret_path': '/kv/somebody', 'secret_key': 'password'}
+
+    For the data to be valid, the keys of this dictionary should correspond
+    with the metadata field (and datatypes) defined in the associated
+    target CredentialType e.g.,
+    """
+
+    def schema(self, credential_type):
+        # determine the defined fields for the associated credential type
+        properties = {}
+        for field in credential_type.inputs.get('metadata', []):
+            field = field.copy()
+            properties[field['id']] = field
+            if field.get('choices', []):
+                field['enum'] = list(field['choices'])[:]
+        return {
+            'type': 'object',
+            'properties': properties,
+            'additionalProperties': False,
+        }
+
+    def validate(self, value, model_instance):
+        if not isinstance(value, dict):
+            return super(DynamicCredentialInputField, self).validate(value, model_instance)
+
+        super(JSONSchemaField, self).validate(value, model_instance)
+        credential_type = model_instance.source_credential.credential_type
+        errors = {}
+        for error in Draft4Validator(
+            self.schema(credential_type),
+            format_checker=self.format_checker
+        ).iter_errors(value):
+            if error.validator == 'pattern' and 'error' in error.schema:
+                error.message = error.schema['error'].format(instance=error.instance)
+            if 'id' not in error.schema:
+                # If the error is not for a specific field, it's specific to
+                # `inputs` in general
+                raise django_exceptions.ValidationError(
+                    error.message,
+                    code='invalid',
+                    params={'value': value},
+                )
+            errors[error.schema['id']] = [error.message]
+
+        defined_metadata = [field.get('id') for field in credential_type.inputs.get('metadata', [])]
+        for field in credential_type.inputs.get('required', []):
+            if field in defined_metadata and not value.get(field, None):
+                errors[field] = [_('required for %s') % (
+                    credential_type.name
+                )]
+
+        if errors:
+            raise serializers.ValidationError({
+                'metadata': errors
+            })
+
+
 class CredentialInputField(JSONSchemaField):
     """
     Used to validate JSON for
@@ -593,8 +656,9 @@ class CredentialInputField(JSONSchemaField):
             errors[error.schema['id']] = [error.message]
 
         inputs = model_instance.credential_type.inputs
+        defined_fields = model_instance.credential_type.defined_fields
         for field in inputs.get('required', []):
-            if not value.get(field, None):
+            if field in defined_fields and not value.get(field, None):
                 errors[field] = [_('required for %s') % (
                     model_instance.credential_type.name
                 )]
@@ -603,7 +667,7 @@ class CredentialInputField(JSONSchemaField):
         # represented without complicated JSON schema
         if (
                 model_instance.credential_type.managed_by_tower is True and
-                'ssh_key_unlock' in model_instance.credential_type.defined_fields
+                'ssh_key_unlock' in defined_fields
         ):
 
             # in order to properly test the necessity of `ssh_key_unlock`, we

awx/main/migrations/0067_v350_credential_plugins.py

@@ -7,6 +7,7 @@ import django.db.models.deletion
 import taggit.managers
 
 # AWX
+import awx.main.fields
 from awx.main.models import CredentialType
 
 
@@ -31,6 +32,7 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('description', models.TextField(blank=True, default='')),
                 ('input_field_name', models.CharField(max_length=1024)),
+                ('metadata', awx.main.fields.DynamicCredentialInputField(blank=True, default={})),
                 ('created_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_created+", to=settings.AUTH_USER_MODEL)),
                 ('modified_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_modified+", to=settings.AUTH_USER_MODEL)),
                 ('source_credential', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='target_input_source', to='main.Credential')),

awx/main/models/credential/__init__.py

@@ -23,7 +23,8 @@ from django.utils.encoding import force_text
 from awx.api.versioning import reverse
 from awx.main.fields import (ImplicitRoleField, CredentialInputField,
                              CredentialTypeInputField,
-                             CredentialTypeInjectorField)
+                             CredentialTypeInjectorField,
+                             DynamicCredentialInputField,)
 from awx.main.utils import decrypt_field, classproperty
 from awx.main.utils.safe_yaml import safe_dump
 from awx.main.validators import validate_ssh_private_key
@@ -1325,6 +1326,10 @@ class CredentialInputSource(PrimordialModel):
     input_field_name = models.CharField(
         max_length=1024,
     )
+    metadata = DynamicCredentialInputField(
+        blank=True,
+        default={}
+    )
 
     def clean_target_credential(self):
         if self.target_credential.kind == 'external':
@@ -1353,6 +1358,8 @@ class CredentialInputSource(PrimordialModel):
                 backend_kwargs[field_name] = decrypt_field(self.source_credential, field_name)
             else:
                 backend_kwargs[field_name] = value
+
+        backend_kwargs.update(self.metadata)
         return backend(
             self.target_credential.inputs.get(self.input_field_name),
             **backend_kwargs

awx/main/tasks.py

@@ -1283,8 +1283,8 @@ class RunJob(BaseTask):
                 ...
             },
             'certificates': {
-                <awx.main.models.Credential>: <signed SSH certifacte data>,
+                <awx.main.models.Credential>: <signed SSH certificate data>,
-                <awx.main.models.Credential>: <signed SSH certifacte data>,
+                <awx.main.models.Credential>: <signed SSH certificate data>,
                 ...
             }
         }
@@ -2208,8 +2208,8 @@ class RunAdHocCommand(BaseTask):
                 ...
             },
             'certificates': {
-                <awx.main.models.Credential>: <signed SSH certifacte data>,
+                <awx.main.models.Credential>: <signed SSH certificate data>,
-                <awx.main.models.Credential>: <signed SSH certifacte data>,
+                <awx.main.models.Credential>: <signed SSH certificate data>,
                 ...
             }
         }

awx/main/tests/functional/api/test_credential_input_sources.py

@@ -15,6 +15,7 @@ def test_associate_credential_input_source(get, post, admin, vault_credential, e
     params = {
         'source_credential': external_credential.pk,
         'input_field_name': 'vault_password',
+        'metadata': {'key': 'some_example_key'},
         'associate': True
     }
     response = post(sublist_url, params, admin)
@@ -31,6 +32,8 @@ def test_associate_credential_input_source(get, post, admin, vault_credential, e
         kwargs={'version': 'v2'}
     ), admin).data['count'] == 1
     assert CredentialInputSource.objects.count() == 1
+    input_source = CredentialInputSource.objects.first()
+    assert input_source.metadata == {'key': 'some_example_key'}
 
     # detach
     params = {
@@ -50,6 +53,29 @@ def test_associate_credential_input_source(get, post, admin, vault_credential, e
     assert CredentialInputSource.objects.count() == 0
 
 
+@pytest.mark.django_db
+@pytest.mark.parametrize('metadata', [
+    {},  # key is required
+    {'key': None},  # must be a string
+    {'key': 123},  # must be a string
+    {'extraneous': 'foo'},  # invalid parameter
+])
+def test_associate_credential_input_source_with_invalid_metadata(get, post, admin, vault_credential, external_credential, metadata):
+    sublist_url = reverse(
+        'api:credential_input_source_sublist',
+        kwargs={'version': 'v2', 'pk': vault_credential.pk}
+    )
+
+    params = {
+        'source_credential': external_credential.pk,
+        'input_field_name': 'vault_password',
+        'metadata': metadata,
+        'associate': True
+    }
+    response = post(sublist_url, params, admin)
+    assert response.status_code == 400
+
+
 @pytest.mark.django_db
 def test_cannot_create_from_list(get, post, admin, vault_credential, external_credential):
     params = {
@@ -73,6 +99,7 @@ def test_create_credential_input_source_with_external_target_returns_400(post, a
         'source_credential': external_credential.pk,
         'input_field_name': 'token',
         'associate': True,
+        'metadata': {'key': 'some_key'},
     }
     response = post(sublist_url, params, admin)
     assert response.status_code == 400
@@ -102,7 +129,8 @@ def test_create_credential_input_source_with_undefined_input_returns_400(post, a
     )
     params = {
         'source_credential': external_credential.pk,
-        'input_field_name': 'not_defined_for_credential_type'
+        'input_field_name': 'not_defined_for_credential_type',
+        'metadata': {'key': 'some_key'}
     }
     response = post(sublist_url, params, admin)
     assert response.status_code == 400

awx/main/tests/functional/conftest.py

@@ -265,7 +265,16 @@ def credentialtype_external():
             'secret': True,
             'help_text': 'An access token for the server.'
         }],
-        'required': ['url', 'token'],
+        'metadata': [{
+            'id': 'key',
+            'label': 'Key',
+            'type': 'string'
+        }, {
+            'id': 'version',
+            'label': 'Version',
+            'type': 'string'
+        }],
+        'required': ['url', 'token', 'key'],
     }
     external_type = CredentialType(
         kind='external',

docs/credential_plugins.md

@@ -1,9 +1,175 @@
 Credential Plugins
-================================
+==================
-### Development
+
-```shell
+By default, sensitive credential values (such as SSH passwords, SSH private
-# The vault server will be available at localhost:8200.
+keys, API tokens for cloud services) in AWX are encrypted with a symmetric
-# From within the awx container, the vault server is reachable at hashivault:8200.
+encryption cipher utilizing AES-256 in CBC mode alongside a SHA-256 HMAC and
-# See docker-hashivault-override.yml for the development root token.
+stored in the AWX database.
-make docker-compose-hashivault
+
+Alternatively, AWX supports retrieving secret values from third-party secret
+management systems, such as HashiCorp Vault and Microsoft Azure Key Vault.
+These external secret values will be fetched on demand every time they are
+needed (generally speaking, immediately before running a playbook that needs
+them).
+
+Configuring Secret Lookups
+--------------------------
+
+When configuring AWX to pull a secret from a third party system, there are
+generally three steps.
+
+Here is an example of creating an (1) AWX Machine Credential with
+a static username, `example-user` and (2) an externally sourced secret from
+HashiCorp Vault Key/Value system which will populate the (3) password field on
+the Machine Credential.
+
+1.  Create the Machine Credential with a static username, `example-user`.
+
+    ```shell
+    HTTP POST https://awx.example.org/api/v2/credentials/
+
+    {
+        "organization": X,
+        "credential_type": <primary_key_of_machine_credential_type>,
+        "inputs": {
+            "username": "example-user"
+        }
+    }
+
+2.  Create a second credential used to _authenticate_ with the external
+    secret management system (e.g.,, in this example, specifying a URL and an
+    OAuth2.0 token _to access_ HashiCorp Vault)
+
+    ```shell
+    HTTP POST https://awx.example.org/api/v2/credentials/
+    {
+        "organization": X,
+        "credential_type": <primary_key_of_hashicorp_vault_credential_type>,
+        "inputs": {
+            "url": "https://my-vault.example.org",
+            "token": "some-oauth-token",
+            "api_version": "v2",
+        }
+    }
+
+3.  _Link_ the `password` field for the Machine credential to the external
+    system by specifying the source (in this example, the HashiCorp credential)
+    and metadata about the path (e.g., `/some/path/to/my/password/`).
+
+    ```shell
+    HTTP POST https://awx.example.org/api/v2/credentials/N/input_sources/
+    {
+        "input_field_name": "",
+        "source_credential": <primary_key_of_hashicorp_vault_credential>
+        "metadata": {
+            "secret_path": "/path/to/my/secret/"
+            "secret_field": "password"
+            "secret_version": 92
+        }
+    }
+    ```
+
+Note that you can perform these lookups on *any* credential field - not just
+the `password` field for Machine credentials.  You could just as easily create
+an AWS credential and use lookups to retrieve the Access Key and Secret Key
+from an external secret management system.
+
+Writing Custom Credential Plugins
+---------------------------------
+
+Credential Plugins in AWX are just importable Python functions that are
+registered using setuptools entrypoints
+(https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins)
+
+Example plugins officially supported in AWX can be found in the source code at
+`awx.main.credential_plugins`.
+
+Credential plugins are any Python object which defines attribute lookups for `.name`, `.inputs`, and `.backend`:
+
+```python
+import collections
+
+CredentialPlugin = collections.namedtuple('CredentialPlugin', ['name', 'inputs', 'backend'])
+
+def some_callable(value_from_awx, **kwargs):
+    return some_libary.get_secret_key(
+        url=kwargs['url'],
+        token=kwargs['token'],
+        key=kwargs['secret_key']
+    )
+
+some_fancy_plugin = CredentialPlugin(
+    'My Plugin Name',
+    # inputs will be used to create a new CredentialType() instance
+    #
+    # inputs.fields represents fields the user will specify *when they create*
+    # a credential of this type; they generally represent fields
+    # used for authentication (URL to the credential management system, any
+    # fields necessary for authentication, such as an OAuth2.0 token, or
+    # a username and password). They're the types of values you set up _once_
+    # in AWX
+    #
+    # inputs.metadata represents values the user will specify *every time
+    # they link two credentials together*
+    # this is generally _pathing_ information about _where_ in the external
+    # management system you can find the value you care about i.e.,
+    #
+    # "I would like Machine Credential A to retrieve its username using
+    # Credential-O-Matic B at secret_key=some_key"
+    inputs={
+        'fields': [{
+            'id': 'url',
+            'label': 'Server URL',
+            'type': 'string',
+        }, {
+            'id': 'token',
+            'label': 'Authentication Token',
+            'type': 'string',
+            'secret': True,
+        }],
+        'metadata': [{
+            'id': 'secret_key',
+            'label': 'Secret Key',
+            'type': 'string',
+            'help_text': 'The value of the key in My Credential System to fetch.'
+        }],
+        'required': ['url', 'token', 'secret_key'],
+    },
+    # backend is a callable function which will be passed all of the values
+    # defined in `inputs`; this function is responsible for taking the arguments,
+    # interacting with the third party credential management system in question
+    # using Python code, and returning the value from the third party
+    # credential management system
+    backend = some_callable
+```
+
+Plugins are registered by specifying an entry point in the `setuptools.setup()`
+call (generally in the package's `setup.py` file - https://github.com/ansible/awx/blob/devel/setup.py):
+
+```python
+setuptools.setup(
+    ...,
+    entry_points = {
+        ...,
+        'awx.credential_plugins': [
+            'fancy_plugin = awx.main.credential_plugins.fancy:some_fancy_plugin',
+        ]
+    }
+)
+```
+
+Fetching vs. Transforming Credential Data
+-----------------------------------------
+While _most_ credential plugins will be used to _fetch_ secrets from external
+systems, they can also be used to *transform* data from Tower _using_ an
+external secret management system.  An example use case is generating signed
+public keys:
+
+```python
+def my_key_signer(unsigned_value_from_awx, **kwargs):
+    return some_libary.sign(
+        url=kwargs['url'],
+        token=kwargs['token'],
+        public_data=unsigned_value_from_awx
+    )
 ```