diff --git a/awx/api/serializers.py b/awx/api/serializers.py index ca4262dbed..3f9964fdfc 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2833,6 +2833,7 @@ class CredentialInputSourceSerializer(BaseSerializer): 'type', 'url', 'input_field_name', + 'metadata', 'target_credential', 'source_credential', 'source_credential_type', diff --git a/awx/main/credential_plugins/azure_kv.py b/awx/main/credential_plugins/azure_kv.py index 2fe8c00ec9..be35720e67 100644 --- a/awx/main/credential_plugins/azure_kv.py +++ b/awx/main/credential_plugins/azure_kv.py @@ -22,7 +22,8 @@ azure_keyvault_inputs = { 'id': 'tenant', 'label': 'Tenant ID', 'type': 'string' - }, { + }], + 'metadata': [{ 'id': 'secret_field', 'label': 'Secret Name', 'type': 'string', @@ -33,7 +34,7 @@ azure_keyvault_inputs = { 'type': 'string', 'help_text': 'Used to specify a specific secret version (if left empty, the latest version will be used).', }], - 'required': ['url', 'client', 'secret', 'tenant'], + 'required': ['url', 'client', 'secret', 'tenant', 'secret_field'], } diff --git a/awx/main/credential_plugins/hashivault.py b/awx/main/credential_plugins/hashivault.py index 555de51c27..c7582eadd6 100644 --- a/awx/main/credential_plugins/hashivault.py +++ b/awx/main/credential_plugins/hashivault.py @@ -1,3 +1,4 @@ +import copy import os import pathlib @@ -18,7 +19,8 @@ base_inputs = { 'type': 'string', 'secret': True, 'help_text': 'The access token used to authenticate to the Vault server', - }, { + }], + 'metadata': [{ 'id': 'secret_path', 'label': 'Path to Secret', 'type': 'string', @@ -27,50 +29,49 @@ base_inputs = { 'required': ['url', 'token', 'secret_path'], } -hashi_kv_inputs = { - 'fields': base_inputs['fields'] + [{ - 'id': 'secret_field', - 'label': 'Key Name', - 'type': 'string', - 'help_text': 'The name of the key to look up in the secret.', - }, { - 'id': 'secret_version', - 'label': 'Secret Version (v2 only)', - 'type': 'string', - 'help_text': 'Used to specify a specific secret version (if left empty, the latest version will be used).', - }, { - 'id': 'api_version', - 'label': 'API Version', - 'choices': ['v1', 'v2'], - 'help_text': 'API v1 is for static key/value lookups. API v2 is for versioned key/value lookups.', - 'default': 'v1', - }], - 'required': base_inputs['required'] + ['secret_field', 'api_version'] -} +hashi_kv_inputs = copy.deepcopy(base_inputs) +hashi_kv_inputs['fields'].append({ + 'id': 'api_version', + 'label': 'API Version', + 'choices': ['v1', 'v2'], + 'help_text': 'API v1 is for static key/value lookups. API v2 is for versioned key/value lookups.', + 'default': 'v1', +}) +hashi_kv_inputs['metadata'].extend([{ + 'id': 'secret_key', + 'label': 'Key Name', + 'type': 'string', + 'help_text': 'The name of the key to look up in the secret.', +}, { + 'id': 'secret_version', + 'label': 'Secret Version (v2 only)', + 'type': 'string', + 'help_text': 'Used to specify a specific secret version (if left empty, the latest version will be used).', +}]) +hashi_kv_inputs['required'].extend(['api_version', 'secret_key']) -hashi_ssh_inputs = { - 'fields': base_inputs['fields'] + [{ - 'id': 'role', - 'label': 'Role Name', - 'type': 'string', - 'help_text': 'The name of the role used to sign.' - }, { - 'id': 'valid_principals', - 'label': 'Valid Principals', - 'type': 'string', - 'help_text': 'Valid principals (either usernames or hostnames) that the certificate should be signed for.', - }], - 'required': base_inputs['required'] + ['role'] -} +hashi_ssh_inputs = copy.deepcopy(base_inputs) +hashi_ssh_inputs['metadata'].extend([{ + 'id': 'role', + 'label': 'Role Name', + 'type': 'string', + 'help_text': 'The name of the role used to sign.' +}, { + 'id': 'valid_principals', + 'label': 'Valid Principals', + 'type': 'string', + 'help_text': 'Valid principals (either usernames or hostnames) that the certificate should be signed for.', +}]) +hashi_ssh_inputs['required'].extend(['role']) def kv_backend(raw, **kwargs): token = kwargs['token'] url = kwargs['url'] secret_path = kwargs['secret_path'] - secret_field = kwargs.get('secret_field', None) + secret_key = kwargs.get('secret_key', None) - api_version = kwargs.get('api_version', None) + api_version = kwargs['api_version'] client = Client(url=url, token=token, verify=True) if api_version == 'v2': @@ -99,12 +100,12 @@ def kv_backend(raw, **kwargs): 'could not read secret {} from {}'.format(secret_path, url) ) - if secret_field: + if secret_key: try: - return response['data'][secret_field] + return response['data'][secret_key] except KeyError: raise RuntimeError( - '{} is not present at {}'.format(secret_field, secret_path) + '{} is not present at {}'.format(secret_key, secret_path) ) return response['data'] diff --git a/awx/main/fields.py b/awx/main/fields.py index f8a20738ef..acd8a4f1a5 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -480,6 +480,69 @@ def format_ssh_private_key(value): return True +class DynamicCredentialInputField(JSONSchemaField): + """ + Used to validate JSON for + `awx.main.models.credential:CredentialInputSource().metadata`. + + Metadata for input sources is represented as a dictionary e.g., + {'secret_path': '/kv/somebody', 'secret_key': 'password'} + + For the data to be valid, the keys of this dictionary should correspond + with the metadata field (and datatypes) defined in the associated + target CredentialType e.g., + """ + + def schema(self, credential_type): + # determine the defined fields for the associated credential type + properties = {} + for field in credential_type.inputs.get('metadata', []): + field = field.copy() + properties[field['id']] = field + if field.get('choices', []): + field['enum'] = list(field['choices'])[:] + return { + 'type': 'object', + 'properties': properties, + 'additionalProperties': False, + } + + def validate(self, value, model_instance): + if not isinstance(value, dict): + return super(DynamicCredentialInputField, self).validate(value, model_instance) + + super(JSONSchemaField, self).validate(value, model_instance) + credential_type = model_instance.source_credential.credential_type + errors = {} + for error in Draft4Validator( + self.schema(credential_type), + format_checker=self.format_checker + ).iter_errors(value): + if error.validator == 'pattern' and 'error' in error.schema: + error.message = error.schema['error'].format(instance=error.instance) + if 'id' not in error.schema: + # If the error is not for a specific field, it's specific to + # `inputs` in general + raise django_exceptions.ValidationError( + error.message, + code='invalid', + params={'value': value}, + ) + errors[error.schema['id']] = [error.message] + + defined_metadata = [field.get('id') for field in credential_type.inputs.get('metadata', [])] + for field in credential_type.inputs.get('required', []): + if field in defined_metadata and not value.get(field, None): + errors[field] = [_('required for %s') % ( + credential_type.name + )] + + if errors: + raise serializers.ValidationError({ + 'metadata': errors + }) + + class CredentialInputField(JSONSchemaField): """ Used to validate JSON for @@ -593,8 +656,9 @@ class CredentialInputField(JSONSchemaField): errors[error.schema['id']] = [error.message] inputs = model_instance.credential_type.inputs + defined_fields = model_instance.credential_type.defined_fields for field in inputs.get('required', []): - if not value.get(field, None): + if field in defined_fields and not value.get(field, None): errors[field] = [_('required for %s') % ( model_instance.credential_type.name )] @@ -603,7 +667,7 @@ class CredentialInputField(JSONSchemaField): # represented without complicated JSON schema if ( model_instance.credential_type.managed_by_tower is True and - 'ssh_key_unlock' in model_instance.credential_type.defined_fields + 'ssh_key_unlock' in defined_fields ): # in order to properly test the necessity of `ssh_key_unlock`, we diff --git a/awx/main/migrations/0067_v350_credential_plugins.py b/awx/main/migrations/0067_v350_credential_plugins.py index 0d0e65255b..427660b045 100644 --- a/awx/main/migrations/0067_v350_credential_plugins.py +++ b/awx/main/migrations/0067_v350_credential_plugins.py @@ -7,6 +7,7 @@ import django.db.models.deletion import taggit.managers # AWX +import awx.main.fields from awx.main.models import CredentialType @@ -31,6 +32,7 @@ class Migration(migrations.Migration): ('modified', models.DateTimeField(default=None, editable=False)), ('description', models.TextField(blank=True, default='')), ('input_field_name', models.CharField(max_length=1024)), + ('metadata', awx.main.fields.DynamicCredentialInputField(blank=True, default={})), ('created_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_created+", to=settings.AUTH_USER_MODEL)), ('modified_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_modified+", to=settings.AUTH_USER_MODEL)), ('source_credential', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='target_input_source', to='main.Credential')), diff --git a/awx/main/models/credential/__init__.py b/awx/main/models/credential/__init__.py index eac6fd4d71..f8597ba4dc 100644 --- a/awx/main/models/credential/__init__.py +++ b/awx/main/models/credential/__init__.py @@ -23,7 +23,8 @@ from django.utils.encoding import force_text from awx.api.versioning import reverse from awx.main.fields import (ImplicitRoleField, CredentialInputField, CredentialTypeInputField, - CredentialTypeInjectorField) + CredentialTypeInjectorField, + DynamicCredentialInputField,) from awx.main.utils import decrypt_field, classproperty from awx.main.utils.safe_yaml import safe_dump from awx.main.validators import validate_ssh_private_key @@ -1325,6 +1326,10 @@ class CredentialInputSource(PrimordialModel): input_field_name = models.CharField( max_length=1024, ) + metadata = DynamicCredentialInputField( + blank=True, + default={} + ) def clean_target_credential(self): if self.target_credential.kind == 'external': @@ -1353,6 +1358,8 @@ class CredentialInputSource(PrimordialModel): backend_kwargs[field_name] = decrypt_field(self.source_credential, field_name) else: backend_kwargs[field_name] = value + + backend_kwargs.update(self.metadata) return backend( self.target_credential.inputs.get(self.input_field_name), **backend_kwargs diff --git a/awx/main/tasks.py b/awx/main/tasks.py index cb479779e7..da9dbdb202 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1283,8 +1283,8 @@ class RunJob(BaseTask): ... }, 'certificates': { - : , - : , + : , + : , ... } } @@ -2208,8 +2208,8 @@ class RunAdHocCommand(BaseTask): ... }, 'certificates': { - : , - : , + : , + : , ... } } diff --git a/awx/main/tests/functional/api/test_credential_input_sources.py b/awx/main/tests/functional/api/test_credential_input_sources.py index 4f0cb371a6..516bb54f67 100644 --- a/awx/main/tests/functional/api/test_credential_input_sources.py +++ b/awx/main/tests/functional/api/test_credential_input_sources.py @@ -15,6 +15,7 @@ def test_associate_credential_input_source(get, post, admin, vault_credential, e params = { 'source_credential': external_credential.pk, 'input_field_name': 'vault_password', + 'metadata': {'key': 'some_example_key'}, 'associate': True } response = post(sublist_url, params, admin) @@ -31,6 +32,8 @@ def test_associate_credential_input_source(get, post, admin, vault_credential, e kwargs={'version': 'v2'} ), admin).data['count'] == 1 assert CredentialInputSource.objects.count() == 1 + input_source = CredentialInputSource.objects.first() + assert input_source.metadata == {'key': 'some_example_key'} # detach params = { @@ -50,6 +53,29 @@ def test_associate_credential_input_source(get, post, admin, vault_credential, e assert CredentialInputSource.objects.count() == 0 +@pytest.mark.django_db +@pytest.mark.parametrize('metadata', [ + {}, # key is required + {'key': None}, # must be a string + {'key': 123}, # must be a string + {'extraneous': 'foo'}, # invalid parameter +]) +def test_associate_credential_input_source_with_invalid_metadata(get, post, admin, vault_credential, external_credential, metadata): + sublist_url = reverse( + 'api:credential_input_source_sublist', + kwargs={'version': 'v2', 'pk': vault_credential.pk} + ) + + params = { + 'source_credential': external_credential.pk, + 'input_field_name': 'vault_password', + 'metadata': metadata, + 'associate': True + } + response = post(sublist_url, params, admin) + assert response.status_code == 400 + + @pytest.mark.django_db def test_cannot_create_from_list(get, post, admin, vault_credential, external_credential): params = { @@ -73,6 +99,7 @@ def test_create_credential_input_source_with_external_target_returns_400(post, a 'source_credential': external_credential.pk, 'input_field_name': 'token', 'associate': True, + 'metadata': {'key': 'some_key'}, } response = post(sublist_url, params, admin) assert response.status_code == 400 @@ -102,7 +129,8 @@ def test_create_credential_input_source_with_undefined_input_returns_400(post, a ) params = { 'source_credential': external_credential.pk, - 'input_field_name': 'not_defined_for_credential_type' + 'input_field_name': 'not_defined_for_credential_type', + 'metadata': {'key': 'some_key'} } response = post(sublist_url, params, admin) assert response.status_code == 400 diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index 8dcbc381a4..625ff0c007 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -265,7 +265,16 @@ def credentialtype_external(): 'secret': True, 'help_text': 'An access token for the server.' }], - 'required': ['url', 'token'], + 'metadata': [{ + 'id': 'key', + 'label': 'Key', + 'type': 'string' + }, { + 'id': 'version', + 'label': 'Version', + 'type': 'string' + }], + 'required': ['url', 'token', 'key'], } external_type = CredentialType( kind='external', diff --git a/docs/credential_plugins.md b/docs/credential_plugins.md index 3cf07cf6c5..3067bb6de2 100644 --- a/docs/credential_plugins.md +++ b/docs/credential_plugins.md @@ -1,9 +1,175 @@ Credential Plugins -================================ -### Development -```shell -# The vault server will be available at localhost:8200. -# From within the awx container, the vault server is reachable at hashivault:8200. -# See docker-hashivault-override.yml for the development root token. -make docker-compose-hashivault +================== + +By default, sensitive credential values (such as SSH passwords, SSH private +keys, API tokens for cloud services) in AWX are encrypted with a symmetric +encryption cipher utilizing AES-256 in CBC mode alongside a SHA-256 HMAC and +stored in the AWX database. + +Alternatively, AWX supports retrieving secret values from third-party secret +management systems, such as HashiCorp Vault and Microsoft Azure Key Vault. +These external secret values will be fetched on demand every time they are +needed (generally speaking, immediately before running a playbook that needs +them). + +Configuring Secret Lookups +-------------------------- + +When configuring AWX to pull a secret from a third party system, there are +generally three steps. + +Here is an example of creating an (1) AWX Machine Credential with +a static username, `example-user` and (2) an externally sourced secret from +HashiCorp Vault Key/Value system which will populate the (3) password field on +the Machine Credential. + +1. Create the Machine Credential with a static username, `example-user`. + + ```shell + HTTP POST https://awx.example.org/api/v2/credentials/ + + { + "organization": X, + "credential_type": , + "inputs": { + "username": "example-user" + } + } + +2. Create a second credential used to _authenticate_ with the external + secret management system (e.g.,, in this example, specifying a URL and an + OAuth2.0 token _to access_ HashiCorp Vault) + + ```shell + HTTP POST https://awx.example.org/api/v2/credentials/ + { + "organization": X, + "credential_type": , + "inputs": { + "url": "https://my-vault.example.org", + "token": "some-oauth-token", + "api_version": "v2", + } + } + +3. _Link_ the `password` field for the Machine credential to the external + system by specifying the source (in this example, the HashiCorp credential) + and metadata about the path (e.g., `/some/path/to/my/password/`). + + ```shell + HTTP POST https://awx.example.org/api/v2/credentials/N/input_sources/ + { + "input_field_name": "", + "source_credential": + "metadata": { + "secret_path": "/path/to/my/secret/" + "secret_field": "password" + "secret_version": 92 + } + } + ``` + +Note that you can perform these lookups on *any* credential field - not just +the `password` field for Machine credentials. You could just as easily create +an AWS credential and use lookups to retrieve the Access Key and Secret Key +from an external secret management system. + +Writing Custom Credential Plugins +--------------------------------- + +Credential Plugins in AWX are just importable Python functions that are +registered using setuptools entrypoints +(https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins) + +Example plugins officially supported in AWX can be found in the source code at +`awx.main.credential_plugins`. + +Credential plugins are any Python object which defines attribute lookups for `.name`, `.inputs`, and `.backend`: + +```python +import collections + +CredentialPlugin = collections.namedtuple('CredentialPlugin', ['name', 'inputs', 'backend']) + +def some_callable(value_from_awx, **kwargs): + return some_libary.get_secret_key( + url=kwargs['url'], + token=kwargs['token'], + key=kwargs['secret_key'] + ) + +some_fancy_plugin = CredentialPlugin( + 'My Plugin Name', + # inputs will be used to create a new CredentialType() instance + # + # inputs.fields represents fields the user will specify *when they create* + # a credential of this type; they generally represent fields + # used for authentication (URL to the credential management system, any + # fields necessary for authentication, such as an OAuth2.0 token, or + # a username and password). They're the types of values you set up _once_ + # in AWX + # + # inputs.metadata represents values the user will specify *every time + # they link two credentials together* + # this is generally _pathing_ information about _where_ in the external + # management system you can find the value you care about i.e., + # + # "I would like Machine Credential A to retrieve its username using + # Credential-O-Matic B at secret_key=some_key" + inputs={ + 'fields': [{ + 'id': 'url', + 'label': 'Server URL', + 'type': 'string', + }, { + 'id': 'token', + 'label': 'Authentication Token', + 'type': 'string', + 'secret': True, + }], + 'metadata': [{ + 'id': 'secret_key', + 'label': 'Secret Key', + 'type': 'string', + 'help_text': 'The value of the key in My Credential System to fetch.' + }], + 'required': ['url', 'token', 'secret_key'], + }, + # backend is a callable function which will be passed all of the values + # defined in `inputs`; this function is responsible for taking the arguments, + # interacting with the third party credential management system in question + # using Python code, and returning the value from the third party + # credential management system + backend = some_callable +``` + +Plugins are registered by specifying an entry point in the `setuptools.setup()` +call (generally in the package's `setup.py` file - https://github.com/ansible/awx/blob/devel/setup.py): + +```python +setuptools.setup( + ..., + entry_points = { + ..., + 'awx.credential_plugins': [ + 'fancy_plugin = awx.main.credential_plugins.fancy:some_fancy_plugin', + ] + } +) +``` + +Fetching vs. Transforming Credential Data +----------------------------------------- +While _most_ credential plugins will be used to _fetch_ secrets from external +systems, they can also be used to *transform* data from Tower _using_ an +external secret management system. An example use case is generating signed +public keys: + +```python +def my_key_signer(unsigned_value_from_awx, **kwargs): + return some_libary.sign( + url=kwargs['url'], + token=kwargs['token'], + public_data=unsigned_value_from_awx + ) ```