From a36a53fe40147ad5cb0118c742d1f88b4089be83 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 19 Apr 2017 15:19:29 -0400 Subject: [PATCH 1/2] implement CredentialType env, file, and extra_vars injectors see: #5877 --- awx/main/fields.py | 18 -- awx/main/models/credential.py | 89 +++++++++ awx/main/models/inventory.py | 2 +- awx/main/tasks.py | 9 + awx/main/tests/functional/test_credential.py | 11 -- awx/main/tests/unit/test_tasks.py | 194 +++++++++++++++++++ 6 files changed, 293 insertions(+), 30 deletions(-) diff --git a/awx/main/fields.py b/awx/main/fields.py index ab4f2cb1c5..2f2fd66e08 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -699,24 +699,6 @@ class CredentialTypeInjectorField(JSONSchemaField): 'additionalProperties': False, 'required': ['template'], }, - 'ssh': { - 'type': 'object', - 'properties': { - 'private': {'type': 'string'}, - 'public': {'type': 'string'}, - }, - 'additionalProperties': False, - 'required': ['public', 'private'], - }, - 'password': { - 'type': 'object', - 'properties': { - 'key': {'type': 'string'}, - 'value': {'type': 'string'}, - }, - 'additionalProperties': False, - 'required': ['key', 'value'], - }, 'env': { 'type': 'object', 'patternProperties': { diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index e16fddb611..390c3f413f 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -2,7 +2,14 @@ # All Rights Reserved. from collections import OrderedDict import functools +import json import operator +import os +import stat +import tempfile + +# Jinja2 +from jinja2 import Template # Django from django.db import models @@ -539,6 +546,88 @@ class CredentialType(CommonModelNameNotUnique): match = cls.objects.filter(**requirements)[:1].get() return match + def inject_credential(self, credential, env, safe_env, args, safe_args, private_data_dir): + """ + Inject credential data into the environment variables and arguments + passed to `ansible-playbook` + + :param credential: a :class:`awx.main.models.Credential` instance + :param env: a dictionary of environment variables used in + the `ansible-playbook` call. This method adds + additional environment variables based on + custom `env` injectors defined on this + CredentialType. + :param safe_env: a dictionary of environment variables stored + in the database for the job run + (`UnifiedJob.job_env`); secret values should + be stripped + :param args: a list of arguments passed to + `ansible-playbook` in the style of + `subprocess.call(args)`. This method appends + additional arguments based on custom + `extra_vars` injectors defined on this + CredentialType. + :param safe_args: a list of arguments stored in the database for + the job run (`UnifiedJob.job_args`); secret + values should be stripped + :param private_data_dir: a temporary directory to store files generated + by `file` injectors (like config files or key + files) + """ + if not self.injectors: + return + + class TowerNamespace: + filename = None + + tower_namespace = TowerNamespace() + + # maintain a normal namespace for building the ansible-playbook arguments (env and args) + namespace = {'tower': tower_namespace} + + # maintain a sanitized namespace for building the DB-stored arguments (safe_env and safe_args) + safe_namespace = {'tower': tower_namespace} + + # build a normal namespace with secret values decrypted (for + # ansible-playbook) and a safe namespace with secret values hidden (for + # DB storage) + for field_name, value in credential.inputs.items(): + if field_name in self.secret_fields: + value = decrypt_field(credential, field_name) + safe_namespace[field_name] = '**********' + elif len(value): + safe_namespace[field_name] = value + if len(value): + namespace[field_name] = value + + file_tmpl = self.injectors.get('file', {}).get('template') + if file_tmpl is not None: + # If a file template is provided, render the file and update the + # special `tower` template namespace so the filename can be + # referenced in other injectors + data = Template(file_tmpl).render(**namespace) + _, path = tempfile.mkstemp(dir=private_data_dir) + with open(path, 'w') as f: + f.write(data) + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + namespace['tower'].filename = path + + for env_var, tmpl in self.injectors.get('env', {}).items(): + env[env_var] = Template(tmpl).render(**namespace) + safe_env[env_var] = Template(tmpl).render(**safe_namespace) + + extra_vars = {} + safe_extra_vars = {} + for var_name, tmpl in self.injectors.get('extra_vars', {}).items(): + extra_vars[var_name] = Template(tmpl).render(**namespace) + safe_extra_vars[var_name] = Template(tmpl).render(**safe_namespace) + + if extra_vars: + args.extend(['-e', json.dumps(extra_vars)]) + + if safe_extra_vars: + safe_args.extend(['-e', json.dumps(safe_extra_vars)]) + @CredentialType.default def ssh(cls): diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index ae1701c686..4cb049f445 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -979,7 +979,7 @@ class InventorySourceOptions(BaseModel): if not self.source: return None cred = self.credential - if cred: + if cred and self.source != 'custom': # If a credential was provided, it's important that it matches # the actual inventory source being used (Amazon requires Amazon # credentials; Rackspace requires Rackspace credentials; etc...) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 02d875c327..9f80b2f986 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -705,6 +705,15 @@ class BaseTask(Task): cwd = self.build_cwd(instance, **kwargs) env = self.build_env(instance, **kwargs) safe_env = self.build_safe_env(env, **kwargs) + + # handle custom injectors specified on the CredentialType + for type_ in ('credential', 'cloud_credential', 'network_credential'): + credential = getattr(instance, type_, None) + if credential: + credential.credential_type.inject_credential( + credential, env, safe_env, args, safe_args, kwargs['private_data_dir'] + ) + stdout_handle = self.get_stdout_handle(instance) if self.should_use_proot(instance, **kwargs): if not check_proot_installed(): diff --git a/awx/main/tests/functional/test_credential.py b/awx/main/tests/functional/test_credential.py index 908992e623..87e17034a8 100644 --- a/awx/main/tests/functional/test_credential.py +++ b/awx/main/tests/functional/test_credential.py @@ -94,17 +94,6 @@ def test_cred_type_input_schema_validity(input_, valid): ({'file': {}}, False), ({'file': {'template': '{{username}}'}}, True), ({'file': {'foo': 'bar'}}, False), - ({'ssh': 123}, False), - ({'ssh': {}}, False), - ({'ssh': {'public': 'PUB'}}, False), - ({'ssh': {'private': 'PRIV'}}, False), - ({'ssh': {'public': 'PUB', 'private': 'PRIV'}}, True), - ({'ssh': {'public': 'PUB', 'private': 'PRIV', 'a': 'b'}}, False), - ({'password': {}}, False), - ({'password': {'key': 'Password:'}}, False), - ({'password': {'value': '{{pass}}'}}, False), - ({'password': {'key': 'Password:', 'value': '{{pass}}'}}, True), - ({'password': {'key': 'Password:', 'value': '{{pass}}', 'a': 'b'}}, False), ({'env': 123}, False), ({'env': {}}, True), ({'env': {'AWX_SECRET': '{{awx_secret}}'}}, True), diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 5b2fed17d8..53ae7fd645 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -2,6 +2,7 @@ from contextlib import contextmanager from datetime import datetime from functools import partial import ConfigParser +import json import tempfile import pytest @@ -563,6 +564,199 @@ class TestJobCredentials(TestJobExecution): self.task.run_pexpect = mock.Mock(side_effect=run_pexpect_side_effect) self.task.run(self.pk) + def test_custom_environment_injectors_with_jinja_syntax_error(self): + some_cloud = CredentialType( + kind='cloud', + name='SomeCloud', + managed_by_tower=False, + inputs={ + 'fields': [{ + 'id': 'api_token', + 'label': 'API Token', + 'type': 'string' + }] + }, + injectors={ + 'env': { + 'MY_CLOUD_API_TOKEN': '{{api_token.foo()}}' + } + } + ) + self.instance.cloud_credential = Credential( + credential_type=some_cloud, + inputs = {'api_token': 'ABC123'} + ) + with pytest.raises(Exception): + self.task.run(self.pk) + + def test_custom_environment_injectors(self): + some_cloud = CredentialType( + kind='cloud', + name='SomeCloud', + managed_by_tower=False, + inputs={ + 'fields': [{ + 'id': 'api_token', + 'label': 'API Token', + 'type': 'string' + }] + }, + injectors={ + 'env': { + 'MY_CLOUD_API_TOKEN': '{{api_token}}' + } + } + ) + self.instance.cloud_credential = Credential( + credential_type=some_cloud, + inputs = {'api_token': 'ABC123'} + ) + self.task.run(self.pk) + + assert self.task.run_pexpect.call_count == 1 + call_args, _ = self.task.run_pexpect.call_args_list[0] + job, args, cwd, env, passwords, stdout = call_args + + assert env['MY_CLOUD_API_TOKEN'] == 'ABC123' + + def test_custom_environment_injectors_with_secret_field(self): + some_cloud = CredentialType( + kind='cloud', + name='SomeCloud', + managed_by_tower=False, + inputs={ + 'fields': [{ + 'id': 'password', + 'label': 'Password', + 'type': 'string', + 'secret': True + }] + }, + injectors={ + 'env': { + 'MY_CLOUD_PRIVATE_VAR': '{{password}}' + } + } + ) + self.instance.cloud_credential = Credential( + credential_type=some_cloud, + inputs = {'password': 'SUPER-SECRET-123'} + ) + self.instance.cloud_credential.inputs['password'] = encrypt_field( + self.instance.cloud_credential, 'password' + ) + self.task.run(self.pk) + + assert self.task.run_pexpect.call_count == 1 + call_args, _ = self.task.run_pexpect.call_args_list[0] + job, args, cwd, env, passwords, stdout = call_args + + assert env['MY_CLOUD_PRIVATE_VAR'] == 'SUPER-SECRET-123' + assert 'SUPER-SECRET-123' not in json.dumps(self.task.update_model.call_args_list) + + def test_custom_environment_injectors_with_extra_vars(self): + some_cloud = CredentialType( + kind='cloud', + name='SomeCloud', + managed_by_tower=False, + inputs={ + 'fields': [{ + 'id': 'api_token', + 'label': 'API Token', + 'type': 'string' + }] + }, + injectors={ + 'extra_vars': { + 'api_token': '{{api_token}}' + } + } + ) + self.instance.cloud_credential = Credential( + credential_type=some_cloud, + inputs = {'api_token': 'ABC123'} + ) + self.task.run(self.pk) + + assert self.task.run_pexpect.call_count == 1 + call_args, _ = self.task.run_pexpect.call_args_list[0] + job, args, cwd, env, passwords, stdout = call_args + + assert '-e {"api_token": "ABC123"}' in ' '.join(args) + + def test_custom_environment_injectors_with_secret_extra_vars(self): + """ + extra_vars that contain secret field values should be censored in the DB + """ + some_cloud = CredentialType( + kind='cloud', + name='SomeCloud', + managed_by_tower=False, + inputs={ + 'fields': [{ + 'id': 'password', + 'label': 'Password', + 'type': 'string', + 'secret': True + }] + }, + injectors={ + 'extra_vars': { + 'password': '{{password}}' + } + } + ) + self.instance.cloud_credential = Credential( + credential_type=some_cloud, + inputs = {'password': 'SUPER-SECRET-123'} + ) + self.instance.cloud_credential.inputs['password'] = encrypt_field( + self.instance.cloud_credential, 'password' + ) + self.task.run(self.pk) + + assert self.task.run_pexpect.call_count == 1 + call_args, _ = self.task.run_pexpect.call_args_list[0] + job, args, cwd, env, passwords, stdout = call_args + + assert '-e {"password": "SUPER-SECRET-123"}' in ' '.join(args) + assert 'SUPER-SECRET-123' not in json.dumps(self.task.update_model.call_args_list) + + def test_custom_environment_injectors_with_file(self): + some_cloud = CredentialType( + kind='cloud', + name='SomeCloud', + managed_by_tower=False, + inputs={ + 'fields': [{ + 'id': 'api_token', + 'label': 'API Token', + 'type': 'string' + }] + }, + injectors={ + 'file': { + 'template': '[mycloud]\n{{api_token}}' + }, + 'env': { + 'MY_CLOUD_INI_FILE': '{{tower.filename}}' + } + } + ) + self.instance.cloud_credential = Credential( + credential_type=some_cloud, + inputs = {'api_token': 'ABC123'} + ) + self.task.run(self.pk) + + def run_pexpect_side_effect(*args, **kwargs): + job, args, cwd, env, passwords, stdout = args + assert open(env['MY_CLOUD_INI_FILE'], 'rb').read() == '[mycloud]\nABC123' + return ['successful', 0] + + self.task.run_pexpect = mock.Mock(side_effect=run_pexpect_side_effect) + self.task.run(self.pk) + class TestProjectUpdateCredentials(TestJobExecution): From aff25c914ecc483cd3316002e144316d7400011b Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 20 Apr 2017 12:44:14 -0400 Subject: [PATCH 2/2] blacklist special env vars from being used in CredentialType injectors see: #5877 --- awx/main/models/credential.py | 10 ++++++++++ awx/main/tests/unit/test_tasks.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 390c3f413f..ec587c4c50 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -457,6 +457,14 @@ class CredentialType(CommonModelNameNotUnique): defaults = OrderedDict() + ENV_BLACKLIST = set(( + 'VIRTUAL_ENV', 'PATH', 'PYTHONPATH', 'PROOT_TMP_DIR', 'JOB_ID', + 'INVENTORY_ID', 'INVENTORY_SOURCE_ID', 'INVENTORY_UPDATE_ID', + 'AD_HOC_COMMAND_ID', 'REST_API_URL', 'REST_API_TOKEN', 'TOWER_HOST', + 'MAX_EVENT_RES', 'CALLBACK_QUEUE', 'CALLBACK_CONNECTION', 'CACHE', + 'JOB_CALLBACK_DEBUG', 'INVENTORY_HOSTVARS', 'FACT_QUEUE', + )) + class Meta: app_label = 'main' ordering = ('kind', 'name') @@ -613,6 +621,8 @@ class CredentialType(CommonModelNameNotUnique): namespace['tower'].filename = path for env_var, tmpl in self.injectors.get('env', {}).items(): + if env_var.startswith('ANSIBLE_') or env_var in self.ENV_BLACKLIST: + continue env[env_var] = Template(tmpl).render(**namespace) safe_env[env_var] = Template(tmpl).render(**safe_namespace) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 53ae7fd645..629d1e88b9 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -619,6 +619,36 @@ class TestJobCredentials(TestJobExecution): assert env['MY_CLOUD_API_TOKEN'] == 'ABC123' + def test_custom_environment_injectors_with_reserved_env_var(self): + some_cloud = CredentialType( + kind='cloud', + name='SomeCloud', + managed_by_tower=False, + inputs={ + 'fields': [{ + 'id': 'api_token', + 'label': 'API Token', + 'type': 'string' + }] + }, + injectors={ + 'env': { + 'JOB_ID': 'reserved' + } + } + ) + self.instance.cloud_credential = Credential( + credential_type=some_cloud, + inputs = {'api_token': 'ABC123'} + ) + self.task.run(self.pk) + + assert self.task.run_pexpect.call_count == 1 + call_args, _ = self.task.run_pexpect.call_args_list[0] + job, args, cwd, env, passwords, stdout = call_args + + assert env['JOB_ID'] == str(self.instance.pk) + def test_custom_environment_injectors_with_secret_field(self): some_cloud = CredentialType( kind='cloud',