From f77298643fba1fe505e892e4fbd2541dc25f0ca2 Mon Sep 17 00:00:00 2001 From: olia-dev Date: Mon, 8 Jul 2019 10:39:42 +0200 Subject: [PATCH 1/6] related #4274 - added option to verify server certificate with a specific CA --- awx/main/credential_plugins/hashivault.py | 70 +++++++++++++++++++---- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/awx/main/credential_plugins/hashivault.py b/awx/main/credential_plugins/hashivault.py index cf9c0944b8..aa4d24b86a 100644 --- a/awx/main/credential_plugins/hashivault.py +++ b/awx/main/credential_plugins/hashivault.py @@ -1,6 +1,9 @@ import copy import os import pathlib +import stat +import tempfile +import threading from urllib.parse import urljoin from .plugin import CredentialPlugin @@ -22,6 +25,12 @@ base_inputs = { 'type': 'string', 'secret': True, 'help_text': _('The access token used to authenticate to the Vault server'), + }, { + 'id': 'cacert', + 'label': _('CA Certificate'), + 'type': 'string', + 'multiline': True, + 'help_text': _('The CA certificate used to verify the SSL certificate of the Vault server') }], 'metadata': [{ 'id': 'secret_path', @@ -73,16 +82,34 @@ hashi_ssh_inputs['metadata'] = [{ hashi_ssh_inputs['required'].extend(['public_key', 'role']) +def create_temporary_fifo(data): + """Open fifo named pipe in a new thread using a temporary file path. The + thread blocks until data is read from the pipe. + Returns the path to the fifo. + :param data(bytes): Data to write to the pipe. + """ + path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names())) + os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR) + + threading.Thread( + target=lambda p, d: open(p, 'wb').write(d), + args=(path, data) + ).start() + return path + + def kv_backend(**kwargs): token = kwargs['token'] url = urljoin(kwargs['url'], 'v1') secret_path = kwargs['secret_path'] secret_key = kwargs.get('secret_key', None) + cacert = kwargs.get('cacert', None) api_version = kwargs['api_version'] sess = requests.Session() sess.headers['Authorization'] = 'Bearer {}'.format(token) + if api_version == 'v2': params = {} if kwargs.get('secret_version'): @@ -93,16 +120,27 @@ def kv_backend(**kwargs): except Exception: mount_point, path = secret_path, [] # https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version - response = sess.get( - '/'.join([url, mount_point, 'data'] + path).rstrip('/'), - params=params, - timeout=30 - ) + if cacert: + response = sess.get( + '/'.join([url, mount_point, 'data'] + path).rstrip('/'), + params=params, + verify=create_temporary_fifo(cacert.encode()) , + timeout=30 + ) + else: + response = sess.get( + '/'.join([url, mount_point, 'data'] + path).rstrip('/'), + params=params, + timeout=30 + ) response.raise_for_status() json = response.json()['data'] else: # https://www.vaultproject.io/api/secret/kv/kv-v1.html#read-secret - response = sess.get('/'.join([url, secret_path]).rstrip('/'), timeout=30) + if cacert: + response = sess.get('/'.join([url, secret_path]).rstrip('/'), verify=create_temporary_fifo(cacert.encode()), timeout=30) + else: + response = sess.get('/'.join([url, secret_path]).rstrip('/'), timeout=30) response.raise_for_status() json = response.json() @@ -121,6 +159,7 @@ def ssh_backend(**kwargs): url = urljoin(kwargs['url'], 'v1') secret_path = kwargs['secret_path'] role = kwargs['role'] + cacert = kwargs.get('cacert', None) sess = requests.Session() sess.headers['Authorization'] = 'Bearer {}'.format(token) @@ -129,12 +168,21 @@ def ssh_backend(**kwargs): } if kwargs.get('valid_principals'): json['valid_principals'] = kwargs['valid_principals'] + # https://www.vaultproject.io/api/secret/ssh/index.html#sign-ssh-key - resp = sess.post( - '/'.join([url, secret_path, 'sign', role]).rstrip('/'), - json=json, - timeout=30 - ) + if cacert: + resp = sess.post( + '/'.join([url, secret_path, 'sign', role]).rstrip('/'), + json=json, + verify=create_temporary_fifo(cacert.encode()), + timeout=30 + ) + else: + resp = sess.post( + '/'.join([url, secret_path, 'sign', role]).rstrip('/'), + json=json, + timeout=30 + ) resp.raise_for_status() return resp.json()['data']['signed_key'] From 7a5efa1adc85eee85e301d45106710c15eda740b Mon Sep 17 00:00:00 2001 From: olia-dev Date: Tue, 9 Jul 2019 13:59:11 +0200 Subject: [PATCH 2/6] related #4274 - added option to verify server certificate with a specific CA --- awx/main/credential_plugins/hashivault.py | 64 ++++++++--------------- 1 file changed, 23 insertions(+), 41 deletions(-) diff --git a/awx/main/credential_plugins/hashivault.py b/awx/main/credential_plugins/hashivault.py index aa4d24b86a..2b21ccba78 100644 --- a/awx/main/credential_plugins/hashivault.py +++ b/awx/main/credential_plugins/hashivault.py @@ -104,43 +104,33 @@ def kv_backend(**kwargs): secret_path = kwargs['secret_path'] secret_key = kwargs.get('secret_key', None) cacert = kwargs.get('cacert', None) - api_version = kwargs['api_version'] + request_kwargs = { 'timeout': 30 } + if cacert: + request_kwargs['verify'] = create_temporary_fifo(cacert.encode()) + sess = requests.Session() sess.headers['Authorization'] = 'Bearer {}'.format(token) if api_version == 'v2': - params = {} if kwargs.get('secret_version'): - params['version'] = kwargs['secret_version'] + request_kwargs['params'] = { 'version': kwargs['secret_version'] } try: mount_point, *path = pathlib.Path(secret_path.lstrip(os.sep)).parts - '/'.join(path) + '/'.join(path) except Exception: mount_point, path = secret_path, [] # https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version - if cacert: - response = sess.get( - '/'.join([url, mount_point, 'data'] + path).rstrip('/'), - params=params, - verify=create_temporary_fifo(cacert.encode()) , - timeout=30 - ) - else: - response = sess.get( - '/'.join([url, mount_point, 'data'] + path).rstrip('/'), - params=params, - timeout=30 - ) + request_url = '/'.join([url, mount_point, 'data'] + path).rstrip('/') + response = sess.get(request_url, **request_kwargs) + response.raise_for_status() json = response.json()['data'] else: - # https://www.vaultproject.io/api/secret/kv/kv-v1.html#read-secret - if cacert: - response = sess.get('/'.join([url, secret_path]).rstrip('/'), verify=create_temporary_fifo(cacert.encode()), timeout=30) - else: - response = sess.get('/'.join([url, secret_path]).rstrip('/'), timeout=30) + request_url = '/'.join([url, secret_path]).rstrip('/') + response = sess.get(request_url, **request_kwargs) + response.raise_for_status() json = response.json() @@ -161,28 +151,20 @@ def ssh_backend(**kwargs): role = kwargs['role'] cacert = kwargs.get('cacert', None) + request_kwargs = { 'timeout': 30 } + if cacert: + request_kwargs['verify'] = create_temporary_fifo(cacert.encode()) + + request_kwargs['json'] = { 'public_key': kwargs['public_key'] } + if kwargs.get('valid_principals'): + request_kwargs['json']['valid_principals'] = kwargs['valid_principals'] + sess = requests.Session() sess.headers['Authorization'] = 'Bearer {}'.format(token) - json = { - 'public_key': kwargs['public_key'] - } - if kwargs.get('valid_principals'): - json['valid_principals'] = kwargs['valid_principals'] - # https://www.vaultproject.io/api/secret/ssh/index.html#sign-ssh-key - if cacert: - resp = sess.post( - '/'.join([url, secret_path, 'sign', role]).rstrip('/'), - json=json, - verify=create_temporary_fifo(cacert.encode()), - timeout=30 - ) - else: - resp = sess.post( - '/'.join([url, secret_path, 'sign', role]).rstrip('/'), - json=json, - timeout=30 - ) + request_url = '/'.join([url, secret_path, 'sign', role]).rstrip('/') + resp = sess.post(request_url, **request_kwargs) + resp.raise_for_status() return resp.json()['data']['signed_key'] From 2ee03b552db1e82602f29812ae6d714ae83253a8 Mon Sep 17 00:00:00 2001 From: olia-dev Date: Tue, 9 Jul 2019 14:14:29 +0200 Subject: [PATCH 3/6] related #4274 - added option to verify server certificate with a specific CA (fixed errors) --- awx/main/credential_plugins/hashivault.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/main/credential_plugins/hashivault.py b/awx/main/credential_plugins/hashivault.py index 2b21ccba78..3f12aad4b4 100644 --- a/awx/main/credential_plugins/hashivault.py +++ b/awx/main/credential_plugins/hashivault.py @@ -106,7 +106,7 @@ def kv_backend(**kwargs): cacert = kwargs.get('cacert', None) api_version = kwargs['api_version'] - request_kwargs = { 'timeout': 30 } + request_kwargs = {'timeout': 30} if cacert: request_kwargs['verify'] = create_temporary_fifo(cacert.encode()) @@ -115,7 +115,7 @@ def kv_backend(**kwargs): if api_version == 'v2': if kwargs.get('secret_version'): - request_kwargs['params'] = { 'version': kwargs['secret_version'] } + request_kwargs['params'] = {'version': kwargs['secret_version']} try: mount_point, *path = pathlib.Path(secret_path.lstrip(os.sep)).parts '/'.join(path) @@ -151,11 +151,11 @@ def ssh_backend(**kwargs): role = kwargs['role'] cacert = kwargs.get('cacert', None) - request_kwargs = { 'timeout': 30 } + request_kwargs = {'timeout': 30} if cacert: request_kwargs['verify'] = create_temporary_fifo(cacert.encode()) - request_kwargs['json'] = { 'public_key': kwargs['public_key'] } + request_kwargs['json'] = {'public_key': kwargs['public_key']} if kwargs.get('valid_principals'): request_kwargs['json']['valid_principals'] = kwargs['valid_principals'] From 7b390fa2fcb733ad297343a5bbc37a3629526e4e Mon Sep 17 00:00:00 2001 From: olia-dev Date: Wed, 10 Jul 2019 11:28:02 +0200 Subject: [PATCH 4/6] related #4274 - moved function 'create_temporary_fifo' to 'awx/main/utils/common.py' and referenced it in other plugins Signed-off-by: olia-dev --- awx/main/credential_plugins/aim.py | 22 ++++------------------ awx/main/credential_plugins/conjur.py | 23 +++++------------------ awx/main/credential_plugins/hashivault.py | 21 ++++----------------- awx/main/utils/common.py | 18 +++++++++++++++++- 4 files changed, 30 insertions(+), 54 deletions(-) diff --git a/awx/main/credential_plugins/aim.py b/awx/main/credential_plugins/aim.py index 8fde480b73..ae466c96ed 100644 --- a/awx/main/credential_plugins/aim.py +++ b/awx/main/credential_plugins/aim.py @@ -9,6 +9,10 @@ from urllib.parse import quote, urlencode, urljoin from django.utils.translation import ugettext_lazy as _ import requests +# AWX +from awx.main.utils import ( + create_temporary_fifo, +) aim_inputs = { 'fields': [{ @@ -60,24 +64,6 @@ aim_inputs = { } -def create_temporary_fifo(data): - """Open fifo named pipe in a new thread using a temporary file path. The - thread blocks until data is read from the pipe. - - Returns the path to the fifo. - - :param data(bytes): Data to write to the pipe. - """ - path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names())) - os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR) - - threading.Thread( - target=lambda p, d: open(p, 'wb').write(d), - args=(path, data) - ).start() - return path - - def aim_backend(**kwargs): url = kwargs['url'] client_cert = kwargs.get('client_cert', None) diff --git a/awx/main/credential_plugins/conjur.py b/awx/main/credential_plugins/conjur.py index c2a60f72b6..8bb40be779 100644 --- a/awx/main/credential_plugins/conjur.py +++ b/awx/main/credential_plugins/conjur.py @@ -10,6 +10,11 @@ from urllib.parse import urljoin, quote_plus from django.utils.translation import ugettext_lazy as _ import requests +# AWX +from awx.main.utils import ( + create_temporary_fifo, +) + conjur_inputs = { 'fields': [{ @@ -51,24 +56,6 @@ conjur_inputs = { } -def create_temporary_fifo(data): - """Open fifo named pipe in a new thread using a temporary file path. The - thread blocks until data is read from the pipe. - - Returns the path to the fifo. - - :param data(bytes): Data to write to the pipe. - """ - path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names())) - os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR) - - threading.Thread( - target=lambda p, d: open(p, 'wb').write(d), - args=(path, data) - ).start() - return path - - def conjur_backend(**kwargs): url = kwargs['url'] api_key = kwargs['api_key'] diff --git a/awx/main/credential_plugins/hashivault.py b/awx/main/credential_plugins/hashivault.py index 3f12aad4b4..f551c3992a 100644 --- a/awx/main/credential_plugins/hashivault.py +++ b/awx/main/credential_plugins/hashivault.py @@ -11,6 +11,10 @@ from .plugin import CredentialPlugin import requests from django.utils.translation import ugettext_lazy as _ +# AWX +from awx.main.utils import ( + create_temporary_fifo, +) base_inputs = { 'fields': [{ @@ -81,23 +85,6 @@ hashi_ssh_inputs['metadata'] = [{ }] hashi_ssh_inputs['required'].extend(['public_key', 'role']) - -def create_temporary_fifo(data): - """Open fifo named pipe in a new thread using a temporary file path. The - thread blocks until data is read from the pipe. - Returns the path to the fifo. - :param data(bytes): Data to write to the pipe. - """ - path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names())) - os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR) - - threading.Thread( - target=lambda p, d: open(p, 'wb').write(d), - args=(path, data) - ).start() - return path - - def kv_backend(**kwargs): token = kwargs['token'] url = urljoin(kwargs['url'], 'v1') diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 3038fbf4fd..b16b142bce 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -44,7 +44,7 @@ __all__ = ['get_object_or_400', 'camelcase_to_underscore', 'underscore_to_camelc 'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict', 'model_instance_diff', 'parse_yaml_or_json', 'RequireDebugTrueOrTest', 'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', 'get_custom_venv_choices', 'get_external_account', - 'task_manager_bulk_reschedule', 'schedule_task_manager', 'classproperty'] + 'task_manager_bulk_reschedule', 'schedule_task_manager', 'classproperty', 'create_temporary_fifo'] def get_object_or_400(klass, *args, **kwargs): @@ -1015,3 +1015,19 @@ class classproperty: def __get__(self, instance, ownerclass): return self.fget(ownerclass) + +def create_temporary_fifo(data): + """Open fifo named pipe in a new thread using a temporary file path. The + thread blocks until data is read from the pipe. + Returns the path to the fifo. + :param data(bytes): Data to write to the pipe. + """ + path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names())) + os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR) + + threading.Thread( + target=lambda p, d: open(p, 'wb').write(d), + args=(path, data) + ).start() + return path + From b2d84a5d8936f32cbd7699b6aab62579353fc3fb Mon Sep 17 00:00:00 2001 From: olia-dev Date: Wed, 10 Jul 2019 12:39:57 +0200 Subject: [PATCH 5/6] related #4274 - moved function 'create_temporary_fifo' to 'awx/main/utils/common.py' and referenced it in other plugins (fixed errors) --- awx/main/credential_plugins/aim.py | 4 ---- awx/main/credential_plugins/conjur.py | 4 ---- awx/main/credential_plugins/hashivault.py | 4 +--- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/awx/main/credential_plugins/aim.py b/awx/main/credential_plugins/aim.py index ae466c96ed..a5bac71fdd 100644 --- a/awx/main/credential_plugins/aim.py +++ b/awx/main/credential_plugins/aim.py @@ -1,9 +1,5 @@ from .plugin import CredentialPlugin -import os -import stat -import tempfile -import threading from urllib.parse import quote, urlencode, urljoin from django.utils.translation import ugettext_lazy as _ diff --git a/awx/main/credential_plugins/conjur.py b/awx/main/credential_plugins/conjur.py index 8bb40be779..55fd2e60f2 100644 --- a/awx/main/credential_plugins/conjur.py +++ b/awx/main/credential_plugins/conjur.py @@ -1,10 +1,6 @@ from .plugin import CredentialPlugin import base64 -import os -import stat -import tempfile -import threading from urllib.parse import urljoin, quote_plus from django.utils.translation import ugettext_lazy as _ diff --git a/awx/main/credential_plugins/hashivault.py b/awx/main/credential_plugins/hashivault.py index f551c3992a..7445f7affd 100644 --- a/awx/main/credential_plugins/hashivault.py +++ b/awx/main/credential_plugins/hashivault.py @@ -1,9 +1,6 @@ import copy import os import pathlib -import stat -import tempfile -import threading from urllib.parse import urljoin from .plugin import CredentialPlugin @@ -85,6 +82,7 @@ hashi_ssh_inputs['metadata'] = [{ }] hashi_ssh_inputs['required'].extend(['public_key', 'role']) + def kv_backend(**kwargs): token = kwargs['token'] url = urljoin(kwargs['url'], 'v1') From 522dcf5ed3e69845c11679332954a86f0eff1beb Mon Sep 17 00:00:00 2001 From: olia-dev Date: Wed, 10 Jul 2019 12:40:26 +0200 Subject: [PATCH 6/6] related #4274 - moved function 'create_temporary_fifo' to 'awx/main/utils/common.py' and referenced it in other plugins (fixed errors) --- awx/main/utils/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index b16b142bce..cf3a511e28 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -1016,6 +1016,7 @@ class classproperty: def __get__(self, instance, ownerclass): return self.fget(ownerclass) + def create_temporary_fifo(data): """Open fifo named pipe in a new thread using a temporary file path. The thread blocks until data is read from the pipe.