From f2f42c2c8a840d19c6e1c92d59af57c88fc78065 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 18 Sep 2017 14:15:03 -0400 Subject: [PATCH 001/141] don't append to the activity stream on LDAP group disassociate for organizations w/ a large number of ldap orgs/teams, this results in a _huge_ number of extraneous activity stream entries see: https://github.com/ansible/ansible-tower/issues/7655 --- awx/sso/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/sso/backends.py b/awx/sso/backends.py index 72b12a6b75..5ed0385018 100644 --- a/awx/sso/backends.py +++ b/awx/sso/backends.py @@ -307,7 +307,7 @@ def _update_m2m_from_groups(user, ldap_user, rel, opts, remove=True): should_add = True if should_add: rel.add(user) - elif remove: + elif remove and user in rel.all(): rel.remove(user) From 554a9586c69dc6aac2782a3e7d2525cca0b1ef19 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 19 Sep 2017 16:29:48 -0400 Subject: [PATCH 002/141] add awx meta variables to adhoc command extra_vars see: https://github.com/ansible/ansible-tower/issues/7513 --- awx/main/tasks.py | 17 ++++++++++-- awx/main/tests/unit/test_tasks.py | 43 +++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 66ba251f40..8a4c6ca27b 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2138,14 +2138,27 @@ class RunAdHocCommand(BaseTask): if ad_hoc_command.verbosity: args.append('-%s' % ('v' * min(5, ad_hoc_command.verbosity))) + # Define special extra_vars for AWX, combine with ad_hoc_command.extra_vars + extra_vars = { + 'tower_job_id': ad_hoc_command.pk, + 'awx_job_id': ad_hoc_command.pk, + } + if ad_hoc_command.created_by: + extra_vars.update({ + 'tower_user_id': ad_hoc_command.created_by.pk, + 'tower_user_name': ad_hoc_command.created_by.username, + 'awx_user_id': ad_hoc_command.created_by.pk, + 'awx_user_name': ad_hoc_command.created_by.username, + }) + if ad_hoc_command.extra_vars_dict: redacted_extra_vars, removed_vars = extract_ansible_vars(ad_hoc_command.extra_vars_dict) if removed_vars: raise ValueError(_( "{} are prohibited from use in ad hoc commands." ).format(", ".join(removed_vars))) - - args.extend(['-e', json.dumps(ad_hoc_command.extra_vars_dict)]) + extra_vars.update(ad_hoc_command.extra_vars_dict) + args.extend(['-e', json.dumps(extra_vars)]) args.extend(['-m', ad_hoc_command.module_name]) args.extend(['-a', ad_hoc_command.module_args]) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 8cb748eb30..ed6c5c9929 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -16,6 +16,7 @@ from django.conf import settings from awx.main.models import ( + AdHocCommand, Credential, CredentialType, Inventory, @@ -26,6 +27,7 @@ from awx.main.models import ( Project, ProjectUpdate, UnifiedJob, + User ) from awx.main import tasks @@ -292,6 +294,18 @@ class TestGenericRun(TestJobExecution): assert '--ro-bind %s %s' % (settings.ANSIBLE_VENV_PATH, settings.ANSIBLE_VENV_PATH) in ' '.join(args) # noqa assert '--ro-bind %s %s' % (settings.AWX_VENV_PATH, settings.AWX_VENV_PATH) in ' '.join(args) # noqa + def test_created_by_extra_vars(self): + self.instance.created_by = User(pk=123, username='angry-spud') + self.task.run(self.pk) + + assert self.run_pexpect.call_count == 1 + call_args, _ = self.run_pexpect.call_args_list[0] + args, cwd, env, stdout = call_args + assert '"tower_user_id": 123,' in ' '.join(args) + assert '"tower_user_name": "angry-spud"' in ' '.join(args) + assert '"awx_user_id": 123,' in ' '.join(args) + assert '"awx_user_name": "angry-spud"' in ' '.join(args) + def test_awx_task_env(self): patch = mock.patch('awx.main.tasks.settings.AWX_TASK_ENV', {'FOO': 'BAR'}) patch.start() @@ -304,6 +318,35 @@ class TestGenericRun(TestJobExecution): assert env['FOO'] == 'BAR' +class TestAdhocRun(TestJobExecution): + + TASK_CLS = tasks.RunAdHocCommand + + def get_instance(self): + return AdHocCommand( + pk=1, + created=datetime.utcnow(), + inventory=Inventory(pk=1), + status='new', + cancel_flag=False, + verbosity=3, + extra_vars={'awx_foo': 'awx-bar'} + ) + + def test_created_by_extra_vars(self): + self.instance.created_by = User(pk=123, username='angry-spud') + self.task.run(self.pk) + + assert self.run_pexpect.call_count == 1 + call_args, _ = self.run_pexpect.call_args_list[0] + args, cwd, env, stdout = call_args + assert '"tower_user_id": 123,' in ' '.join(args) + assert '"tower_user_name": "angry-spud"' in ' '.join(args) + assert '"awx_user_id": 123,' in ' '.join(args) + assert '"awx_user_name": "angry-spud"' in ' '.join(args) + assert '"awx_foo": "awx-bar' in ' '.join(args) + + class TestIsolatedExecution(TestJobExecution): REMOTE_HOST = 'some-isolated-host' From 96572fe3d4c4f130a5be2b771f201836ac94876e Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 19 Sep 2017 15:58:43 -0400 Subject: [PATCH 003/141] don't show polymorphic_ctype in unique validation error messaging see: https://github.com/ansible/ansible-tower/issues/7620 --- awx/main/models/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/awx/main/models/base.py b/awx/main/models/base.py index 6c038dad74..ebfe0bf1c5 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -309,7 +309,10 @@ class PrimordialModel(CreatedModifiedModel): continue if not (self.pk and self.pk == obj.pk): errors.append( - '%s with this (%s) combination already exists.' % (model.__name__, ', '.join(ut)) + '%s with this (%s) combination already exists.' % ( + model.__name__, + ', '.join(set(ut) - {'polymorphic_ctype'}) + ) ) if errors: raise ValidationError(errors) From c8f4320b5820c43770fb4ab2671965aad54344c4 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 20 Sep 2017 14:27:14 -0400 Subject: [PATCH 004/141] allow the credential type to be changed for unused credentials see: https://github.com/ansible/ansible-tower/issues/7607 --- awx/api/serializers.py | 20 +++++-- .../tests/functional/api/test_credential.py | 52 ++++++++++++++----- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index a95a7f0c41..e6b1058778 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2162,10 +2162,22 @@ class CredentialSerializer(BaseSerializer): def validate_credential_type(self, credential_type): if self.instance and credential_type.pk != self.instance.credential_type.pk: - raise ValidationError( - _('You cannot change the credential type of the credential, as it may break the functionality' - ' of the resources using it.'), - ) + for rel in ( + 'ad_hoc_commands', + 'insights_inventories', + 'inventorysources', + 'inventoryupdates', + 'jobs', + 'jobtemplates', + 'projects', + 'projectupdates', + 'workflowjobnodes' + ): + if getattr(self.instance, rel).count() > 0: + raise ValidationError( + _('You cannot change the credential type of the credential, as it may break the functionality' + ' of the resources using it.'), + ) return credential_type diff --git a/awx/main/tests/functional/api/test_credential.py b/awx/main/tests/functional/api/test_credential.py index feebfc08f4..b60d215e14 100644 --- a/awx/main/tests/functional/api/test_credential.py +++ b/awx/main/tests/functional/api/test_credential.py @@ -4,7 +4,9 @@ import re import mock # noqa import pytest -from awx.main.models.credential import Credential, CredentialType +from awx.main.models import (AdHocCommand, Credential, CredentialType, Job, JobTemplate, + Inventory, InventorySource, Project, + WorkflowJobNode) from awx.main.utils import decrypt_field from awx.api.versioning import reverse @@ -1410,7 +1412,17 @@ def test_field_removal(put, organization, admin, credentialtype_ssh, version, pa @pytest.mark.django_db -def test_credential_type_immutable_in_v2(patch, organization, admin, credentialtype_ssh, credentialtype_aws): +@pytest.mark.parametrize('relation, related_obj', [ + ['ad_hoc_commands', AdHocCommand()], + ['insights_inventories', Inventory()], + ['inventorysources', InventorySource()], + ['jobs', Job()], + ['jobtemplates', JobTemplate()], + ['projects', Project()], + ['workflowjobnodes', WorkflowJobNode()], +]) +def test_credential_type_mutability(patch, organization, admin, credentialtype_ssh, + credentialtype_aws, relation, related_obj): cred = Credential( credential_type=credentialtype_ssh, name='Best credential ever', @@ -1422,19 +1434,31 @@ def test_credential_type_immutable_in_v2(patch, organization, admin, credentialt ) cred.save() - response = patch( - reverse('api:credential_detail', kwargs={'version': 'v2', 'pk': cred.pk}), - { - 'credential_type': credentialtype_aws.pk, - 'inputs': { - 'username': u'jim', - 'password': u'pass' - } - }, - admin - ) + related_obj.save() + getattr(cred, relation).add(related_obj) + + def _change_credential_type(): + return patch( + reverse('api:credential_detail', kwargs={'version': 'v2', 'pk': cred.pk}), + { + 'credential_type': credentialtype_aws.pk, + 'inputs': { + 'username': u'jim', + 'password': u'pass' + } + }, + admin + ) + + response = _change_credential_type() assert response.status_code == 400 - assert 'credential_type' in response.data + expected = ['You cannot change the credential type of the credential, ' + 'as it may break the functionality of the resources using it.'] + assert response.data['credential_type'] == expected + + related_obj.delete() + response = _change_credential_type() + assert response.status_code == 200 @pytest.mark.django_db From 7a21a4578129ddc3797cf387d73a5f3551036269 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 20 Sep 2017 15:53:08 -0400 Subject: [PATCH 005/141] properly encode LDAP DN values on validation see: https://github.com/ansible/ansible-tower/issues/7554 --- awx/sso/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/sso/validators.py b/awx/sso/validators.py index 172c21593c..dd1086a426 100644 --- a/awx/sso/validators.py +++ b/awx/sso/validators.py @@ -22,7 +22,7 @@ def validate_ldap_dn(value, with_user=False): else: dn_value = value try: - ldap.dn.str2dn(dn_value) + ldap.dn.str2dn(dn_value.encode('utf-8')) except ldap.DECODING_ERROR: raise ValidationError(_('Invalid DN: %s') % value) From 94d44e879195c8665141f129ea2f8077e33df0b4 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 21 Sep 2017 16:21:29 -0400 Subject: [PATCH 006/141] disable GCE inventory source cache by default, the GCE inventory script caches results on disk for 5 minutes; disable this behavior see: https://github.com/ansible/ansible-tower/issues/7609 --- awx/main/management/commands/inventory_import.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 9959001bf8..c84d12c16c 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -176,6 +176,13 @@ class AnsibleInventoryLoader(object): def load(self): base_args = self.get_base_args() logger.info('Reading Ansible inventory source: %s', self.source) + + # by default, the GCE inventory source caches results on disk for + # 5 minutes; disable this behavior + # https://github.com/ansible/tower/blob/cfb633e8a643b0190fa07b6204b339a1d336cbb3/awx/plugins/inventory/gce.py#L115 + if self.source.endswith('gce.py'): + base_args += ['--refresh-cache'] + data = self.command_to_json(base_args + ['--list']) # TODO: remove after we run custom scripts through ansible-inventory From 94b4dabee2a5a766d6ae301f2476bdbae3f72990 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 22 Sep 2017 09:22:28 -0400 Subject: [PATCH 007/141] disable GCE inventory caching w/ a .ini file see: https://github.com/ansible/ansible-tower/issues/7609 see: https://github.com/ansible/tower/pull/451#pullrequestreview-64454393 --- awx/main/management/commands/inventory_import.py | 6 ------ awx/main/tasks.py | 10 ++++++++++ awx/main/tests/unit/test_tasks.py | 6 ++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index c84d12c16c..797355bf3b 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -177,12 +177,6 @@ class AnsibleInventoryLoader(object): base_args = self.get_base_args() logger.info('Reading Ansible inventory source: %s', self.source) - # by default, the GCE inventory source caches results on disk for - # 5 minutes; disable this behavior - # https://github.com/ansible/tower/blob/cfb633e8a643b0190fa07b6204b339a1d336cbb3/awx/plugins/inventory/gce.py#L115 - if self.source.endswith('gce.py'): - base_args += ['--refresh-cache'] - data = self.command_to_json(base_args + ['--list']) # TODO: remove after we run custom scripts through ansible-inventory diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 8a4c6ca27b..aed99cd9ab 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1881,6 +1881,16 @@ class RunInventoryUpdate(BaseTask): env['GCE_PROJECT'] = passwords.get('source_project', '') env['GCE_PEM_FILE_PATH'] = cloud_credential env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' + + # by default, the GCE inventory source caches results on disk for + # 5 minutes; disable this behavior + cp = ConfigParser.ConfigParser() + cp.add_section('cache') + cp.set('cache', 'cache_max_age', '0') + handle, path = tempfile.mkstemp(dir=kwargs.get('private_data_dir', None)) + cp.write(os.fdopen(handle, 'w')) + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + env['GCE_INI_PATH'] = path elif inventory_update.source == 'openstack': env['OS_CLIENT_CONFIG_FILE'] = cloud_credential elif inventory_update.source == 'satellite6': diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index ed6c5c9929..3a7a218133 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1319,6 +1319,12 @@ class TestInventoryUpdateCredentials(TestJobExecution): assert env['GCE_ZONE'] == expected_gce_zone ssh_key_data = env['GCE_PEM_FILE_PATH'] assert open(ssh_key_data, 'rb').read() == self.EXAMPLE_PRIVATE_KEY + + config = ConfigParser.ConfigParser() + config.read(env['GCE_INI_PATH']) + assert 'cache' in config.sections() + assert config.getint('cache', 'cache_max_age') == 0 + return ['successful', 0] self.run_pexpect.side_effect = run_pexpect_side_effect From 74f250948203ead518f6da3270f6bcea717a2450 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 22 Sep 2017 15:16:15 -0400 Subject: [PATCH 008/141] support ovirt4 as a built-in inventory source see: https://github.com/ansible/ansible-tower/issues/6522 --- awx/main/constants.py | 2 +- ...9_v322_add_support_for_ovirt4_inventory.py | 28 ++ awx/main/migrations/_credentialtypes.py | 3 + awx/main/models/base.py | 2 +- awx/main/models/credential.py | 46 +++ awx/main/models/inventory.py | 6 + awx/main/tests/functional/test_credential.py | 1 + awx/main/tests/unit/test_tasks.py | 35 +++ awx/plugins/inventory/ovirt4.py | 262 ++++++++++++++++++ awx/settings/defaults.py | 10 + tools/docker-compose/Dockerfile | 2 +- 11 files changed, 394 insertions(+), 3 deletions(-) create mode 100644 awx/main/migrations/0009_v322_add_support_for_ovirt4_inventory.py create mode 100755 awx/plugins/inventory/ovirt4.py diff --git a/awx/main/constants.py b/awx/main/constants.py index 4ff22662b3..9c0b01a3d4 100644 --- a/awx/main/constants.py +++ b/awx/main/constants.py @@ -5,7 +5,7 @@ import re from django.utils.translation import ugettext_lazy as _ -CLOUD_PROVIDERS = ('azure_rm', 'ec2', 'gce', 'vmware', 'openstack', 'satellite6', 'cloudforms') +CLOUD_PROVIDERS = ('azure_rm', 'ec2', 'gce', 'vmware', 'openstack', 'ovirt4', 'satellite6', 'cloudforms') SCHEDULEABLE_PROVIDERS = CLOUD_PROVIDERS + ('custom', 'scm',) PRIVILEGE_ESCALATION_METHODS = [ ('sudo', _('Sudo')), ('su', _('Su')), ('pbrun', _('Pbrun')), ('pfexec', _('Pfexec')), ('dzdo', _('DZDO')), ('pmrun', _('Pmrun')), ('runas', _('Runas'))] ANSI_SGR_PATTERN = re.compile(r'\x1b\[[0-9;]*m') diff --git a/awx/main/migrations/0009_v322_add_support_for_ovirt4_inventory.py b/awx/main/migrations/0009_v322_add_support_for_ovirt4_inventory.py new file mode 100644 index 0000000000..e29bdfb4d3 --- /dev/null +++ b/awx/main/migrations/0009_v322_add_support_for_ovirt4_inventory.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +# AWX +from awx.main.migrations import _credentialtypes as credentialtypes + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0008_v320_drop_v1_credential_fields'), + ] + + operations = [ + migrations.RunPython(credentialtypes.create_ovirt4_credtype), + migrations.AlterField( + model_name='inventorysource', + name='source', + field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script'), (b'scm', 'Sourced from a Project'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'ovirt4', 'oVirt4'), (b'custom', 'Custom Script')]), + ), + migrations.AlterField( + model_name='inventoryupdate', + name='source', + field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script'), (b'scm', 'Sourced from a Project'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'ovirt4', 'oVirt4'), (b'custom', 'Custom Script')]), + ), + ] diff --git a/awx/main/migrations/_credentialtypes.py b/awx/main/migrations/_credentialtypes.py index f34d08903a..1c90822bab 100644 --- a/awx/main/migrations/_credentialtypes.py +++ b/awx/main/migrations/_credentialtypes.py @@ -173,3 +173,6 @@ def migrate_job_credentials(apps, schema_editor): finally: utils.get_current_apps = orig_current_apps + +def create_ovirt4_credtype(apps, schema_editor): + CredentialType.defaults['ovirt4']().save() diff --git a/awx/main/models/base.py b/awx/main/models/base.py index ebfe0bf1c5..93bb484a46 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -52,7 +52,7 @@ PROJECT_UPDATE_JOB_TYPE_CHOICES = [ (PERM_INVENTORY_CHECK, _('Check')), ] -CLOUD_INVENTORY_SOURCES = ['ec2', 'vmware', 'gce', 'azure_rm', 'openstack', 'custom', 'satellite6', 'cloudforms', 'scm',] +CLOUD_INVENTORY_SOURCES = ['ec2', 'vmware', 'gce', 'azure_rm', 'openstack', 'ovirt4', 'custom', 'satellite6', 'cloudforms', 'scm',] VERBOSITY_CHOICES = [ (0, '0 (Normal)'), diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 5a3f0b9662..5a1832f323 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -59,6 +59,7 @@ class V1Credential(object): ('gce', 'Google Compute Engine'), ('azure_rm', 'Microsoft Azure Resource Manager'), ('openstack', 'OpenStack'), + ('ovirt4', 'oVirt4'), ('insights', 'Insights'), ] FIELDS = { @@ -1000,3 +1001,48 @@ def insights(cls): }, }, ) + + +@CredentialType.default +def ovirt4(cls): + return cls( + kind='cloud', + name='oVirt4', + managed_by_tower=True, + inputs={ + 'fields': [{ + 'id': 'host', + 'label': 'Host (Authentication URL)', + 'type': 'string', + 'help_text': ('The host to authenticate with.') + }, { + 'id': 'username', + 'label': 'Username', + 'type': 'string' + }, { + 'id': 'password', + 'label': 'Password', + 'type': 'string', + 'secret': True, + }, { + 'id': 'ca_file', + 'label': 'CA File', + 'type': 'string', + 'help_text': ('Absolute file path to the CA file to use (optional)') + }], + 'required': ['host', 'username', 'password'], + }, + injectors={ + 'file': { + 'template': '\n'.join([ + '[ovirt]', + 'ovirt_url={{host}}', + 'ovirt_username={{username}}', + 'ovirt_password={{password}}', + '{% if ca_file %}ovirt_ca_file={{ca_file}}{% endif %}']) + }, + 'env': { + 'OVIRT_INI_PATH': '{{tower.filename}}' + } + }, + ) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 7b5e3d3dfa..f5cd0d8e58 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -872,6 +872,7 @@ class InventorySourceOptions(BaseModel): ('satellite6', _('Red Hat Satellite 6')), ('cloudforms', _('Red Hat CloudForms')), ('openstack', _('OpenStack')), + ('ovirt4', _('oVirt4')), ('custom', _('Custom Script')), ] @@ -1120,6 +1121,11 @@ class InventorySourceOptions(BaseModel): """Red Hat CloudForms region choices (not implemented)""" return [('all', 'All')] + @classmethod + def get_ovirt4_region_choices(self): + """No region supprt""" + return [('all', 'All')] + def clean_credential(self): if not self.source: return None diff --git a/awx/main/tests/functional/test_credential.py b/awx/main/tests/functional/test_credential.py index b4dd0cb0e4..9bcf23e198 100644 --- a/awx/main/tests/functional/test_credential.py +++ b/awx/main/tests/functional/test_credential.py @@ -25,6 +25,7 @@ def test_default_cred_types(): 'insights', 'net', 'openstack', + 'ovirt4', 'satellite6', 'scm', 'ssh', diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 3a7a218133..a9fb3dd448 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -689,6 +689,41 @@ class TestJobCredentials(TestJobExecution): self.run_pexpect.side_effect = run_pexpect_side_effect self.task.run(self.pk) + @pytest.mark.parametrize("ca_file", [None, '/path/to/some/file']) + def test_ovirt4_credentials(self, ca_file): + ovirt4 = CredentialType.defaults['ovirt4']() + inputs = { + 'host': 'some-ovirt-host.example.org', + 'username': 'bob', + 'password': 'some-pass', + } + if ca_file: + inputs['ca_file'] = ca_file + credential = Credential( + pk=1, + credential_type=ovirt4, + inputs=inputs + ) + credential.inputs['password'] = encrypt_field(credential, 'password') + self.instance.extra_credentials.add(credential) + + def run_pexpect_side_effect(*args, **kwargs): + args, cwd, env, stdout = args + config = ConfigParser.ConfigParser() + config.read(env['OVIRT_INI_PATH']) + assert config.get('ovirt', 'ovirt_url') == 'some-ovirt-host.example.org' + assert config.get('ovirt', 'ovirt_username') == 'bob' + assert config.get('ovirt', 'ovirt_password') == 'some-pass' + if ca_file: + assert config.get('ovirt', 'ovirt_ca_file') == ca_file + else: + with pytest.raises(ConfigParser.NoOptionError): + config.get('ovirt', 'ovirt_ca_file') + return ['successful', 0] + + self.run_pexpect.side_effect = run_pexpect_side_effect + self.task.run(self.pk) + def test_net_credentials(self): net = CredentialType.defaults['net']() credential = Credential( diff --git a/awx/plugins/inventory/ovirt4.py b/awx/plugins/inventory/ovirt4.py new file mode 100755 index 0000000000..6221325f34 --- /dev/null +++ b/awx/plugins/inventory/ovirt4.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +# + +""" +oVirt dynamic inventory script +================================= + +Generates dynamic inventory file for oVirt. + +Script will return following attributes for each virtual machine: + - id + - name + - host + - cluster + - status + - description + - fqdn + - os_type + - template + - tags + - statistics + - devices + +When run in --list mode, virtual machines are grouped by the following categories: + - cluster + - tag + - status + + Note: If there is some virtual machine which has has more tags it will be in both tag + records. + +Examples: + # Execute update of system on webserver virtual machine: + + $ ansible -i contrib/inventory/ovirt4.py webserver -m yum -a "name=* state=latest" + + # Get webserver virtual machine information: + + $ contrib/inventory/ovirt4.py --host webserver + +Author: Ondra Machacek (@machacekondra) +""" + +import argparse +import os +import sys + +from collections import defaultdict + +try: + import ConfigParser as configparser +except ImportError: + import configparser + +try: + import json +except ImportError: + import simplejson as json + +try: + import ovirtsdk4 as sdk + import ovirtsdk4.types as otypes +except ImportError: + print('oVirt inventory script requires ovirt-engine-sdk-python >= 4.0.0') + sys.exit(1) + + +def parse_args(): + """ + Create command line parser for oVirt dynamic inventory script. + """ + parser = argparse.ArgumentParser( + description='Ansible dynamic inventory script for oVirt.', + ) + parser.add_argument( + '--list', + action='store_true', + default=True, + help='Get data of all virtual machines (default: True).', + ) + parser.add_argument( + '--host', + help='Get data of virtual machines running on specified host.', + ) + parser.add_argument( + '--pretty', + action='store_true', + default=False, + help='Pretty format (default: False).', + ) + return parser.parse_args() + + +def create_connection(): + """ + Create a connection to oVirt engine API. + """ + # Get the path of the configuration file, by default use + # 'ovirt.ini' file in script directory: + default_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'ovirt.ini', + ) + config_path = os.environ.get('OVIRT_INI_PATH', default_path) + + # Create parser and add ovirt section if it doesn't exist: + config = configparser.SafeConfigParser( + defaults={ + 'ovirt_url': None, + 'ovirt_username': None, + 'ovirt_password': None, + 'ovirt_ca_file': None, + } + ) + if not config.has_section('ovirt'): + config.add_section('ovirt') + config.read(config_path) + + # Create a connection with options defined in ini file: + return sdk.Connection( + url=config.get('ovirt', 'ovirt_url'), + username=config.get('ovirt', 'ovirt_username'), + password=config.get('ovirt', 'ovirt_password'), + ca_file=config.get('ovirt', 'ovirt_ca_file'), + insecure=config.get('ovirt', 'ovirt_ca_file') is None, + ) + + +def get_dict_of_struct(connection, vm): + """ + Transform SDK Vm Struct type to Python dictionary. + """ + if vm is None: + return dict() + + vms_service = connection.system_service().vms_service() + clusters_service = connection.system_service().clusters_service() + vm_service = vms_service.vm_service(vm.id) + devices = vm_service.reported_devices_service().list() + tags = vm_service.tags_service().list() + stats = vm_service.statistics_service().list() + labels = vm_service.affinity_labels_service().list() + groups = clusters_service.cluster_service( + vm.cluster.id + ).affinity_groups_service().list() + + return { + 'id': vm.id, + 'name': vm.name, + 'host': connection.follow_link(vm.host).name if vm.host else None, + 'cluster': connection.follow_link(vm.cluster).name, + 'status': str(vm.status), + 'description': vm.description, + 'fqdn': vm.fqdn, + 'os_type': vm.os.type, + 'template': connection.follow_link(vm.template).name, + 'tags': [tag.name for tag in tags], + 'affinity_labels': [label.name for label in labels], + 'affinity_groups': [ + group.name for group in groups + if vm.name in [vm.name for vm in connection.follow_link(group.vms)] + ], + 'statistics': dict( + (stat.name, stat.values[0].datum) for stat in stats + ), + 'devices': dict( + (device.name, [ip.address for ip in device.ips]) for device in devices if device.ips + ), + 'ansible_host': next((device.ips[0].address for device in devices if device.ips), None) + } + + +def get_data(connection, vm_name=None): + """ + Obtain data of `vm_name` if specified, otherwise obtain data of all vms. + """ + vms_service = connection.system_service().vms_service() + clusters_service = connection.system_service().clusters_service() + + if vm_name: + vm = vms_service.list(search='name=%s' % vm_name) or [None] + data = get_dict_of_struct( + connection=connection, + vm=vm[0], + ) + else: + vms = dict() + data = defaultdict(list) + for vm in vms_service.list(): + name = vm.name + vm_service = vms_service.vm_service(vm.id) + cluster_service = clusters_service.cluster_service(vm.cluster.id) + + # Add vm to vms dict: + vms[name] = get_dict_of_struct(connection, vm) + + # Add vm to cluster group: + cluster_name = connection.follow_link(vm.cluster).name + data['cluster_%s' % cluster_name].append(name) + + # Add vm to tag group: + tags_service = vm_service.tags_service() + for tag in tags_service.list(): + data['tag_%s' % tag.name].append(name) + + # Add vm to status group: + data['status_%s' % vm.status].append(name) + + # Add vm to affinity group: + for group in cluster_service.affinity_groups_service().list(): + if vm.name in [ + v.name for v in connection.follow_link(group.vms) + ]: + data['affinity_group_%s' % group.name].append(vm.name) + + # Add vm to affinity label group: + affinity_labels_service = vm_service.affinity_labels_service() + for label in affinity_labels_service.list(): + data['affinity_label_%s' % label.name].append(name) + + data["_meta"] = { + 'hostvars': vms, + } + + return data + + +def main(): + args = parse_args() + connection = create_connection() + + print( + json.dumps( + obj=get_data( + connection=connection, + vm_name=args.host, + ), + sort_keys=args.pretty, + indent=args.pretty * 2, + ) + ) + +if __name__ == '__main__': + main() diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index d4747a8078..b35b0ad376 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -821,6 +821,16 @@ OPENSTACK_HOST_FILTER = r'^.+$' OPENSTACK_EXCLUDE_EMPTY_GROUPS = True OPENSTACK_INSTANCE_ID_VAR = 'openstack.id' +# --------------------- +# ----- oVirt4 ----- +# --------------------- +OVIRT4_ENABLED_VAR = 'status' +OVIRT4_ENABLED_VALUE = 'up' +OVIRT4_GROUP_FILTER = r'^.+$' +OVIRT4_HOST_FILTER = r'^.+$' +OVIRT4_EXCLUDE_EMPTY_GROUPS = True +OVIRT4_INSTANCE_ID_VAR = 'id' + # --------------------- # ----- Foreman ----- # --------------------- diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile index e6fb9c0d43..712c7cd607 100644 --- a/tools/docker-compose/Dockerfile +++ b/tools/docker-compose/Dockerfile @@ -14,7 +14,7 @@ requirements/requirements_dev_uninstall.txt \ RUN yum -y update && yum -y install curl epel-release RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - RUN yum -y localinstall http://download.postgresql.org/pub/repos/yum/9.4/redhat/rhel-6-x86_64/pgdg-centos94-9.4-3.noarch.rpm -RUN yum -y update && yum -y install openssh-server ansible mg vim tmux git mercurial subversion python-devel python-psycopg2 make postgresql postgresql-devel nginx nodejs python-psutil libxml2-devel libxslt-devel libstdc++.so.6 gcc cyrus-sasl-devel cyrus-sasl openldap-devel libffi-devel zeromq-devel python-pip xmlsec1-devel swig krb5-devel xmlsec1-openssl xmlsec1 xmlsec1-openssl-devel libtool-ltdl-devel rabbitmq-server bubblewrap zanata-python-client gettext gcc-c++ bzip2 +RUN yum -y update && yum -y install openssh-server ansible mg vim tmux git mercurial subversion python-devel python-psycopg2 make postgresql postgresql-devel nginx nodejs python-psutil libxml2-devel libxslt-devel libstdc++.so.6 gcc cyrus-sasl-devel cyrus-sasl openldap-devel libffi-devel zeromq-devel python-pip xmlsec1-devel swig krb5-devel xmlsec1-openssl xmlsec1 xmlsec1-openssl-devel libtool-ltdl-devel rabbitmq-server bubblewrap zanata-python-client gettext gcc-c++ libcurl-devel python-pycurl bzip2 RUN pip install virtualenv RUN /usr/bin/ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa RUN mkdir -p /data/db From 7438062b9736882bf23d9ac8ebe88d71e09a7084 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 26 Sep 2017 09:46:03 -0400 Subject: [PATCH 009/141] add ovirt sdk dependency for ovirt4 support --- requirements/requirements_ansible.in | 1 + requirements/requirements_ansible.txt | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in index c686e9545f..b4ab5f60f6 100644 --- a/requirements/requirements_ansible.in +++ b/requirements/requirements_ansible.in @@ -17,6 +17,7 @@ backports.ssl-match-hostname==3.5.0.1 kombu==3.0.37 boto==2.46.1 boto3==1.4.4 +ovirt-engine-sdk-python==4.1.6 python-memcached==1.58 psphere==0.5.2 psutil==5.2.2 diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index 20c57308e8..944b95f72a 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -44,7 +44,7 @@ decorator==4.0.11 # via shade deprecation==1.0.1 # via openstacksdk #docutils==0.12 # via botocore dogpile.cache==0.6.3 # via python-ironicclient, shade -enum34==1.1.6 # via cryptography, msrest +enum34==1.1.6 # via cryptography, msrest, ovirt-engine-sdk-python funcsigs==1.0.2 # via debtcollector, oslo.utils functools32==3.2.3.post2 # via jsonschema futures==3.1.1 # via azure-storage, s3transfer, shade @@ -78,6 +78,7 @@ oslo.config==4.6.0 # via python-keystoneclient oslo.i18n==3.15.3 # via osc-lib, oslo.config, oslo.utils, python-cinderclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient oslo.serialization==2.18.0 # via python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient oslo.utils==3.26.0 # via osc-lib, oslo.serialization, python-cinderclient, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient +ovirt-engine-sdk-python==4.1.6 packaging==16.8 # via setuptools pbr==3.1.1 # via cliff, debtcollector, keystoneauth1, openstacksdk, osc-lib, oslo.i18n, oslo.serialization, oslo.utils, positional, python-cinderclient, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient, requestsexceptions, shade, stevedore positional==1.1.1 # via keystoneauth1, python-keystoneclient @@ -114,7 +115,7 @@ s3transfer==0.1.10 # via boto3 secretstorage==2.3.1 shade==1.20.0 simplejson==3.11.1 # via osc-lib, python-cinderclient, python-neutronclient, python-novaclient -six==1.10.0 # via cliff, cmd2, cryptography, debtcollector, keystoneauth1, munch, ntlm-auth, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, packaging, pyopenssl, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, pyvmomi, pywinrm, setuptools, shade, stevedore, warlock +six==1.10.0 # via cliff, cmd2, cryptography, debtcollector, keystoneauth1, munch, ntlm-auth, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, ovirt-engine-sdk-python, packaging, pyopenssl, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, pyvmomi, pywinrm, setuptools, shade, stevedore, warlock stevedore==1.23.0 # via cliff, keystoneauth1, openstacksdk, osc-lib, oslo.config, python-designateclient, python-keystoneclient suds==0.4 # via psphere tabulate==0.7.7 # via azure-cli-core From e06d4d77343b3ec5680d4c664761c4ae0bc18ed5 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 26 Sep 2017 11:12:48 -0400 Subject: [PATCH 010/141] don't install pycurl from pypi; use a system package instead the ovirt4 sdk relies on pycurl, which is complicated to install w/ pip; rely on pycurl to be provided by a system package instead --- requirements/requirements_ansible_uninstall.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/requirements_ansible_uninstall.txt b/requirements/requirements_ansible_uninstall.txt index 963eac530b..30d8dedee3 100644 --- a/requirements/requirements_ansible_uninstall.txt +++ b/requirements/requirements_ansible_uninstall.txt @@ -1 +1,2 @@ certifi +pycurl # requires system package version From f4ab979b594f98e7e216b50930c8755f805fe3ba Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Tue, 26 Sep 2017 16:04:52 -0400 Subject: [PATCH 011/141] Prevent slugify username from social sso backends Relates #7684 of ansible-tower. Slugify username in python-social-auth means disallowing any non-alphanumerial characters, which is an over-kill for awx/tower, thus disabling it. Signed-off-by: Aaron Tan --- awx/settings/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index b35b0ad376..c1d6a053c9 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -519,7 +519,7 @@ SOCIAL_AUTH_INACTIVE_USER_URL = '/sso/inactive/' SOCIAL_AUTH_RAISE_EXCEPTIONS = False SOCIAL_AUTH_USERNAME_IS_FULL_EMAIL = False -SOCIAL_AUTH_SLUGIFY_USERNAMES = True +#SOCIAL_AUTH_SLUGIFY_USERNAMES = True SOCIAL_AUTH_CLEAN_USERNAMES = True SOCIAL_AUTH_SANITIZE_REDIRECTS = True From f4a252a331a8fcc6292799b9352cada3d707812e Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 26 Sep 2017 14:14:46 -0400 Subject: [PATCH 012/141] add new credential types in a more stable way in migrations instead of writing individual migrations for new built-in credential types, this change makes the "setup_tower_managed_defaults" function idempotent so that it only adds the credential types you're missing --- awx/main/migrations/_credentialtypes.py | 2 +- awx/main/models/credential.py | 8 ++++++++ awx/main/tests/functional/api/test_credential.py | 11 +++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/awx/main/migrations/_credentialtypes.py b/awx/main/migrations/_credentialtypes.py index 1c90822bab..104caa334a 100644 --- a/awx/main/migrations/_credentialtypes.py +++ b/awx/main/migrations/_credentialtypes.py @@ -175,4 +175,4 @@ def migrate_job_credentials(apps, schema_editor): def create_ovirt4_credtype(apps, schema_editor): - CredentialType.defaults['ovirt4']().save() + CredentialType.setup_tower_managed_defaults() diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 5a1832f323..7db8db04e9 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -3,6 +3,7 @@ from collections import OrderedDict import functools import json +import logging import operator import os import stat @@ -35,6 +36,8 @@ from awx.main.utils import encrypt_field __all__ = ['Credential', 'CredentialType', 'V1Credential'] +logger = logging.getLogger('awx.main.models.credential') + class V1Credential(object): @@ -468,6 +471,11 @@ class CredentialType(CommonModelNameNotUnique): for default in cls.defaults.values(): default_ = default() if persisted: + if CredentialType.objects.filter(name=default_.name, kind=default_.kind).count(): + continue + logger.debug(_( + "adding %s credential type" % default_.name + )) default_.save() @classmethod diff --git a/awx/main/tests/functional/api/test_credential.py b/awx/main/tests/functional/api/test_credential.py index b60d215e14..ff29f9ed3f 100644 --- a/awx/main/tests/functional/api/test_credential.py +++ b/awx/main/tests/functional/api/test_credential.py @@ -14,6 +14,17 @@ EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY- EXAMPLE_ENCRYPTED_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nxyz==\n-----END PRIVATE KEY-----' +@pytest.mark.django_db +def test_idempotent_credential_type_setup(): + assert CredentialType.objects.count() == 0 + CredentialType.setup_tower_managed_defaults() + total = CredentialType.objects.count() + assert total > 0 + + CredentialType.setup_tower_managed_defaults() + assert CredentialType.objects.count() == total + + @pytest.mark.django_db @pytest.mark.parametrize('kind, total', [ ('ssh', 1), ('net', 0) From a11e33458fee23892999b1a37eefb0ffb031d64a Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Wed, 27 Sep 2017 15:11:38 -0400 Subject: [PATCH 013/141] Include Tower configurations into activity stream Relates #7386 of ansible-tower. Due to the uniqueness of Tower configuration datastore model, it is not fully compatible with activity stream workflow. This PR introduced setting field for activitystream model along with other changes to make Tower configuration a special case for activity streams. Signed-off-by: Aaron Tan --- awx/api/serializers.py | 7 +++++++ awx/conf/registry.py | 3 +++ awx/conf/utils.py | 12 ++++++++++- ...2_add_setting_field_for_activity_stream.py | 20 ++++++++++++++++++ awx/main/models/activity_stream.py | 3 +++ awx/main/signals.py | 12 +++++++++-- .../functional/api/test_activity_streams.py | 21 +++++++++++++++++++ 7 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 awx/main/migrations/0010_v322_add_setting_field_for_activity_stream.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index e6b1058778..fbc9db29f5 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -3797,6 +3797,11 @@ class ActivityStreamSerializer(BaseSerializer): if fk == 'schedule': rel['unified_job_template'] = thisItem.unified_job_template.get_absolute_url(self.context.get('request')) + if obj.setting and obj.setting.get('category', None): + rel['setting'] = self.reverse( + 'api:setting_singleton_detail', + kwargs={'category_slug': obj.setting['category']} + ) return rel def _get_rel(self, obj, fk): @@ -3848,6 +3853,8 @@ class ActivityStreamSerializer(BaseSerializer): username = obj.actor.username, first_name = obj.actor.first_name, last_name = obj.actor.last_name) + if obj.setting: + summary_fields['setting'] = [obj.setting] return summary_fields diff --git a/awx/conf/registry.py b/awx/conf/registry.py index a50753aecb..4e07f0d7da 100644 --- a/awx/conf/registry.py +++ b/awx/conf/registry.py @@ -120,6 +120,9 @@ class SettingsRegistry(object): def is_setting_read_only(self, setting): return bool(self._registry.get(setting, {}).get('read_only', False)) + def get_setting_category(self, setting): + return self._registry.get(setting, {}).get('category_slug', None) + def get_setting_field(self, setting, mixin_class=None, for_user=False, **kwargs): from rest_framework.fields import empty field_kwargs = {} diff --git a/awx/conf/utils.py b/awx/conf/utils.py index b780038e9f..b0d4ffb694 100755 --- a/awx/conf/utils.py +++ b/awx/conf/utils.py @@ -9,7 +9,10 @@ import shutil # RedBaron from redbaron import RedBaron, indent -__all__ = ['comment_assignments'] +# AWX +from awx.conf.registry import settings_registry + +__all__ = ['comment_assignments', 'conf_to_dict'] def comment_assignments(patterns, assignment_names, dry_run=True, backup_suffix='.old'): @@ -103,6 +106,13 @@ def comment_assignments_in_file(filename, assignment_names, dry_run=True, backup return '\n'.join(diff_lines) +def conf_to_dict(obj): + return { + 'category': settings_registry.get_setting_category(obj.key), + 'name': obj.key, + } + + if __name__ == '__main__': pattern = os.path.join(os.path.dirname(__file__), '..', 'settings', 'local_*.py') diffs = comment_assignments(pattern, ['AUTH_LDAP_ORGANIZATION_MAP']) diff --git a/awx/main/migrations/0010_v322_add_setting_field_for_activity_stream.py b/awx/main/migrations/0010_v322_add_setting_field_for_activity_stream.py new file mode 100644 index 0000000000..ad3856567a --- /dev/null +++ b/awx/main/migrations/0010_v322_add_setting_field_for_activity_stream.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +import awx.main.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0009_v322_add_support_for_ovirt4_inventory'), + ] + + operations = [ + migrations.AddField( + model_name='activitystream', + name='setting', + field=awx.main.fields.JSONField(default=dict, blank=True), + ), + ] diff --git a/awx/main/models/activity_stream.py b/awx/main/models/activity_stream.py index 4d8a4e9709..94df2f985c 100644 --- a/awx/main/models/activity_stream.py +++ b/awx/main/models/activity_stream.py @@ -3,6 +3,7 @@ # Tower from awx.api.versioning import reverse +from awx.main.fields import JSONField # Django from django.db import models @@ -66,6 +67,8 @@ class ActivityStream(models.Model): role = models.ManyToManyField("Role", blank=True) instance_group = models.ManyToManyField("InstanceGroup", blank=True) + setting = JSONField(blank=True) + def get_absolute_url(self, request=None): return reverse('api:activity_stream_detail', kwargs={'pk': self.pk}, request=request) diff --git a/awx/main/signals.py b/awx/main/signals.py index 77fd91a5c3..1936f96c76 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -26,6 +26,8 @@ from awx.main.fields import is_implicit_parent from awx.main.consumers import emit_channel_notification +from awx.conf.utils import conf_to_dict + __all__ = [] logger = logging.getLogger('awx.main.signals') @@ -402,12 +404,15 @@ def activity_stream_create(sender, instance, created, **kwargs): object1=object1, changes=json.dumps(changes), actor=get_current_user_or_none()) - activity_entry.save() #TODO: Weird situation where cascade SETNULL doesn't work # it might actually be a good idea to remove all of these FK references since # we don't really use them anyway. if instance._meta.model_name != 'setting': # Is not conf.Setting instance + activity_entry.save() getattr(activity_entry, object1).add(instance) + else: + activity_entry.setting = conf_to_dict(instance) + activity_entry.save() def activity_stream_update(sender, instance, **kwargs): @@ -433,9 +438,12 @@ def activity_stream_update(sender, instance, **kwargs): object1=object1, changes=json.dumps(changes), actor=get_current_user_or_none()) - activity_entry.save() if instance._meta.model_name != 'setting': # Is not conf.Setting instance + activity_entry.save() getattr(activity_entry, object1).add(instance) + else: + activity_entry.setting = conf_to_dict(instance) + activity_entry.save() def activity_stream_delete(sender, instance, **kwargs): diff --git a/awx/main/tests/functional/api/test_activity_streams.py b/awx/main/tests/functional/api/test_activity_streams.py index 1396ac3b48..382add5dc6 100644 --- a/awx/main/tests/functional/api/test_activity_streams.py +++ b/awx/main/tests/functional/api/test_activity_streams.py @@ -5,6 +5,7 @@ from awx.api.versioning import reverse from awx.main.middleware import ActivityStreamMiddleware from awx.main.models.activity_stream import ActivityStream from awx.main.access import ActivityStreamAccess +from awx.conf.models import Setting def mock_feature_enabled(feature): @@ -47,6 +48,26 @@ def test_basic_fields(monkeypatch, organization, get, user, settings): assert response.data['summary_fields']['organization'][0]['name'] == 'test-org' +@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled) +@pytest.mark.django_db +def test_ctint_activity_stream(monkeypatch, get, user, settings): + Setting.objects.create(key="FOO", value="bar") + settings.ACTIVITY_STREAM_ENABLED = True + u = user('admin', True) + activity_stream = ActivityStream.objects.filter(setting={'name': 'FOO', 'category': None}).latest('pk') + activity_stream.actor = u + activity_stream.save() + + aspk = activity_stream.pk + url = reverse('api:activity_stream_detail', kwargs={'pk': aspk}) + response = get(url, user('admin', True)) + + assert response.status_code == 200 + assert 'summary_fields' in response.data + assert 'setting' in response.data['summary_fields'] + assert response.data['summary_fields']['setting'][0]['name'] == 'FOO' + + @mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled) @pytest.mark.django_db def test_middleware_actor_added(monkeypatch, post, get, user, settings): From 9ee18d02c84ef1a0c3a0f48eceefd553ac734c75 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Sat, 19 Aug 2017 07:53:27 -0400 Subject: [PATCH 014/141] new method of performance logging --- awx/api/generics.py | 1 - awx/main/middleware.py | 36 ++++++++++++++++++++++++++++++++++++ awx/main/utils/formatters.py | 5 ++++- awx/settings/defaults.py | 5 +++++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/awx/api/generics.py b/awx/api/generics.py index d2d7f0ff40..36176016b0 100644 --- a/awx/api/generics.py +++ b/awx/api/generics.py @@ -139,7 +139,6 @@ class APIView(views.APIView): response['X-API-Query-Count'] = len(q_times) response['X-API-Query-Time'] = '%0.3fs' % sum(q_times) - analytics_logger.info("api response", extra=dict(python_objects=dict(request=request, response=response))) return response def get_authenticate_header(self, request): diff --git a/awx/main/middleware.py b/awx/main/middleware.py index dfb08b20fb..0c71eeed92 100644 --- a/awx/main/middleware.py +++ b/awx/main/middleware.py @@ -5,6 +5,10 @@ import logging import threading import uuid import six +import time +import cProfile +import pstats +import os from django.conf import settings from django.contrib.auth.models import User @@ -23,6 +27,38 @@ from awx.conf import fields, register logger = logging.getLogger('awx.main.middleware') analytics_logger = logging.getLogger('awx.analytics.activity_stream') +perf_logger = logging.getLogger('awx.analytics.performance') + + +class TimingMiddleware(threading.local): + + dest = '/var/lib/awx/profile' + + def process_request(self, request): + self.start_time = time.time() + if settings.AWX_REQUEST_PROFILE: + self.prof = cProfile.Profile() + self.prof.enable() + + def process_response(self, request, response): + total_time = time.time() - self.start_time + response['X-API-Total-Time'] = '%0.3fs' % total_time + if settings.AWX_REQUEST_PROFILE: + self.prof.disable() + cprofile_file = self.save_profile_file(request) + response['cprofile_file'] = cprofile_file + perf_logger.info('api response times', extra=dict(python_objects=dict(request=request, response=response))) + return response + + def save_profile_file(self, request): + if not os.path.isdir(self.dest): + os.makedirs(self.dest) + filename = '%.3fs-%s' % (pstats.Stats(self.prof).total_tt, uuid.uuid4()) + filepath = os.path.join(self.dest, filename) + with open(filepath, 'w') as f: + f.write('%s %s\n' % (request.method, request.get_full_path())) + pstats.Stats(self.prof, stream=f).sort_stats('cumulative').print_stats() + return filepath class ActivityStreamMiddleware(threading.local): diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py index 12034dd5ea..380641fdf9 100644 --- a/awx/main/utils/formatters.py +++ b/awx/main/utils/formatters.py @@ -120,6 +120,7 @@ class LogstashFormatter(LogstashFormatterVersion1): # exist if SQL_DEBUG is turned on in settings. headers = [ (float, 'X-API-Time'), # may end with an 's' "0.33s" + (float, 'X-API-Total-Time'), (int, 'X-API-Query-Count'), (float, 'X-API-Query-Time'), # may also end with an 's' (str, 'X-API-Node'), @@ -131,9 +132,11 @@ class LogstashFormatter(LogstashFormatterVersion1): 'path': request.path, 'path_info': request.path_info, 'query_string': request.META['QUERY_STRING'], - 'data': request.data, } + if hasattr(request, 'data'): + data_for_log['request']['data'] = request.data + return data_for_log def get_extra_fields(self, record): diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index c1d6a053c9..9f25ddb525 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -211,6 +211,7 @@ TEMPLATE_CONTEXT_PROCESSORS = ( # NOQA ) MIDDLEWARE_CLASSES = ( # NOQA + 'awx.main.middleware.TimingMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.middleware.common.CommonMiddleware', @@ -1134,4 +1135,8 @@ LOGGING = { }, } } +# Apply coloring to messages logged to the console COLOR_LOGS = False + +# Use middleware to get request statistics +AWX_REQUEST_PROFILE = False From a0cfbb93e9325672c6d195cd2d634fa52248cb3f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 2 Oct 2017 14:30:26 -0400 Subject: [PATCH 015/141] fix busted 3.2.2 activity stream migration see: ansible/ansible-tower#7704 --- awx/main/migrations/0007_v320_data_migrations.py | 3 ++- ...0009_v322_add_setting_field_for_activity_stream.py} | 2 +- ...y => 0010_v322_add_support_for_ovirt4_inventory.py} | 2 +- awx/main/migrations/__init__.py | 10 ++++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) rename awx/main/migrations/{0010_v322_add_setting_field_for_activity_stream.py => 0009_v322_add_setting_field_for_activity_stream.py} (86%) rename awx/main/migrations/{0009_v322_add_support_for_ovirt4_inventory.py => 0010_v322_add_support_for_ovirt4_inventory.py} (95%) diff --git a/awx/main/migrations/0007_v320_data_migrations.py b/awx/main/migrations/0007_v320_data_migrations.py index 9461e81bcb..e8ede86ba9 100644 --- a/awx/main/migrations/0007_v320_data_migrations.py +++ b/awx/main/migrations/0007_v320_data_migrations.py @@ -6,6 +6,7 @@ from __future__ import unicode_literals from django.db import migrations, models # AWX +from awx.main.migrations import ActivityStreamDisabledMigration from awx.main.migrations import _inventory_source as invsrc from awx.main.migrations import _migration_utils as migration_utils from awx.main.migrations import _reencrypt as reencrypt @@ -15,7 +16,7 @@ from awx.main.migrations import _azure_credentials as azurecreds import awx.main.fields -class Migration(migrations.Migration): +class Migration(ActivityStreamDisabledMigration): dependencies = [ ('main', '0006_v320_release'), diff --git a/awx/main/migrations/0010_v322_add_setting_field_for_activity_stream.py b/awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py similarity index 86% rename from awx/main/migrations/0010_v322_add_setting_field_for_activity_stream.py rename to awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py index ad3856567a..3d69de2b33 100644 --- a/awx/main/migrations/0010_v322_add_setting_field_for_activity_stream.py +++ b/awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py @@ -8,7 +8,7 @@ import awx.main.fields class Migration(migrations.Migration): dependencies = [ - ('main', '0009_v322_add_support_for_ovirt4_inventory'), + ('main', '0008_v320_drop_v1_credential_fields'), ] operations = [ diff --git a/awx/main/migrations/0009_v322_add_support_for_ovirt4_inventory.py b/awx/main/migrations/0010_v322_add_support_for_ovirt4_inventory.py similarity index 95% rename from awx/main/migrations/0009_v322_add_support_for_ovirt4_inventory.py rename to awx/main/migrations/0010_v322_add_support_for_ovirt4_inventory.py index e29bdfb4d3..e6db61e4b8 100644 --- a/awx/main/migrations/0009_v322_add_support_for_ovirt4_inventory.py +++ b/awx/main/migrations/0010_v322_add_support_for_ovirt4_inventory.py @@ -10,7 +10,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('main', '0008_v320_drop_v1_credential_fields'), + ('main', '0009_v322_add_setting_field_for_activity_stream'), ] operations = [ diff --git a/awx/main/migrations/__init__.py b/awx/main/migrations/__init__.py index 709b95a6a6..2ea54e7880 100644 --- a/awx/main/migrations/__init__.py +++ b/awx/main/migrations/__init__.py @@ -1,2 +1,12 @@ # Copyright (c) 2016 Ansible, Inc. # All Rights Reserved. + +from django.db.migrations import Migration + + +class ActivityStreamDisabledMigration(Migration): + + def apply(self, project_state, schema_editor, collect_sql=False): + from awx.main.signals import disable_activity_stream + with disable_activity_stream(): + return Migration.apply(self, project_state, schema_editor, collect_sql) From 4be4e3db7f36f092da28f3992ee4a4341bec95dc Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 28 Sep 2017 14:21:18 -0400 Subject: [PATCH 016/141] encrypt job survey data see: https://github.com/ansible/ansible-tower/issues/7046 --- awx/api/views.py | 15 ++- awx/main/models/jobs.py | 7 +- awx/main/models/mixins.py | 15 +++ awx/main/tasks.py | 2 +- .../tests/functional/api/test_survey_spec.py | 105 ++++++++++++++++++ .../tests/unit/models/test_survey_models.py | 1 + awx/main/tests/unit/test_tasks.py | 16 ++- awx/main/utils/encryption.py | 9 +- 8 files changed, 164 insertions(+), 6 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index ba7fc709e1..366ee29957 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -15,6 +15,7 @@ import logging import requests from base64 import b64encode from collections import OrderedDict +import six # Django from django.conf import settings @@ -72,6 +73,7 @@ from awx.main.utils import ( extract_ansible_vars, decrypt_field, ) +from awx.main.utils.encryption import encrypt_value from awx.main.utils.filters import SmartFilter from awx.main.utils.insights import filter_insights_api_response @@ -2899,8 +2901,15 @@ class JobTemplateSurveySpec(GenericAPIView): if "required" not in survey_item: return Response(dict(error=_("'required' missing from survey question %s.") % str(idx)), status=status.HTTP_400_BAD_REQUEST) - if survey_item["type"] == "password": - if survey_item.get("default") and survey_item["default"].startswith('$encrypted$'): + if survey_item["type"] == "password" and "default" in survey_item: + if not isinstance(survey_item['default'], six.string_types): + return Response( + _("Value %s for '%s' expected to be a string." % ( + survey_item["default"], survey_item["variable"] + )), + status=status.HTTP_400_BAD_REQUEST + ) + elif survey_item["default"].startswith('$encrypted$'): if not obj.survey_spec: return Response(dict(error=_("$encrypted$ is reserved keyword and may not be used as a default for password {}.".format(str(idx)))), status=status.HTTP_400_BAD_REQUEST) @@ -2909,6 +2918,8 @@ class JobTemplateSurveySpec(GenericAPIView): for old_item in old_spec['spec']: if old_item['variable'] == survey_item['variable']: survey_item['default'] = old_item['default'] + else: + survey_item['default'] = encrypt_value(survey_item['default']) idx += 1 obj.survey_spec = new_spec diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index b9e6a00359..249c051f20 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -37,6 +37,7 @@ from awx.main.utils import ( ignore_inventory_computed_fields, parse_yaml_or_json, ) +from awx.main.utils.encryption import encrypt_value from awx.main.fields import ImplicitRoleField from awx.main.models.mixins import ResourceMixin, SurveyJobTemplateMixin, SurveyJobMixin, TaskManagerJobMixin from awx.main.models.base import PERM_INVENTORY_SCAN @@ -385,6 +386,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour # Sort the runtime fields allowed and disallowed by job template ignored_fields = {} prompted_fields = {} + survey_password_variables = self.survey_password_variables() ask_for_vars_dict = self._ask_for_vars_dict() @@ -402,7 +404,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour extra_vars = parse_yaml_or_json(kwargs[field]) for key in extra_vars: if key in survey_vars: - prompted_fields[field][key] = extra_vars[key] + if key in survey_password_variables: + prompted_fields[field][key] = encrypt_value(extra_vars[key]) + else: + prompted_fields[field][key] = extra_vars[key] else: ignored_fields[field][key] = extra_vars[key] else: diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index e13b56fa87..9bdf7194d7 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -13,6 +13,7 @@ from awx.main.models.rbac import ( Role, RoleAncestorEntry, get_roles_on_resource ) from awx.main.utils import parse_yaml_or_json +from awx.main.utils.encryption import decrypt_value, get_encryption_key from awx.main.fields import JSONField @@ -263,6 +264,20 @@ class SurveyJobMixin(models.Model): else: return self.extra_vars + def decrypted_extra_vars(self): + ''' + Decrypts fields marked as passwords in survey. + ''' + if self.survey_passwords: + extra_vars = json.loads(self.extra_vars) + for key in self.survey_passwords: + if key in extra_vars: + value = extra_vars[key] + extra_vars[key] = decrypt_value(get_encryption_key('value', pk=None), value) + return json.dumps(extra_vars) + else: + return self.extra_vars + class TaskManagerUnifiedJobMixin(models.Model): class Meta: diff --git a/awx/main/tasks.py b/awx/main/tasks.py index aed99cd9ab..5dbadd4131 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1180,7 +1180,7 @@ class RunJob(BaseTask): if kwargs.get('display', False) and job.job_template: extra_vars.update(json.loads(job.display_extra_vars())) else: - extra_vars.update(job.extra_vars_dict) + extra_vars.update(json.loads(job.decrypted_extra_vars())) args.extend(['-e', json.dumps(extra_vars)]) # Add path to playbook (relative to project.local_path). diff --git a/awx/main/tests/functional/api/test_survey_spec.py b/awx/main/tests/functional/api/test_survey_spec.py index f679d6806b..0b9a16cea1 100644 --- a/awx/main/tests/functional/api/test_survey_spec.py +++ b/awx/main/tests/functional/api/test_survey_spec.py @@ -91,6 +91,111 @@ def test_survey_spec_sucessful_creation(survey_spec_factory, job_template, post, assert updated_jt.survey_spec == survey_input_data +@mock.patch('awx.api.views.feature_enabled', lambda feature: True) +@pytest.mark.django_db +@pytest.mark.parametrize('value, status', [ + ('SUPERSECRET', 201), + (['some', 'invalid', 'list'], 400), + ({'some-invalid': 'dict'}, 400), + (False, 400) +]) +def test_survey_spec_passwords_are_encrypted_on_launch(job_template_factory, post, admin_user, value, status): + objects = job_template_factory('jt', organization='org1', project='prj', + inventory='inv', credential='cred') + job_template = objects.job_template + job_template.survey_enabled = True + job_template.save() + input_data = { + 'description': 'A survey', + 'spec': [{ + 'index': 0, + 'question_name': 'What is your password?', + 'required': True, + 'variable': 'secret_value', + 'type': 'password' + }], + 'name': 'my survey' + } + post(url=reverse('api:job_template_survey_spec', kwargs={'pk': job_template.id}), + data=input_data, user=admin_user, expect=200) + resp = post(reverse('api:job_template_launch', kwargs={'pk': job_template.pk}), + dict(extra_vars=dict(secret_value=value)), admin_user, expect=status) + + if status == 201: + job = Job.objects.get(pk=resp.data['id']) + assert json.loads(job.extra_vars)['secret_value'].startswith('$encrypted$') + assert json.loads(job.decrypted_extra_vars()) == { + 'secret_value': value + } + else: + assert "for 'secret_value' expected to be a string." in json.dumps(resp.data) + + +@mock.patch('awx.api.views.feature_enabled', lambda feature: True) +@pytest.mark.django_db +@pytest.mark.parametrize('default, status', [ + ('SUPERSECRET', 200), + (['some', 'invalid', 'list'], 400), + ({'some-invalid': 'dict'}, 400), + (False, 400) +]) +def test_survey_spec_default_passwords_are_encrypted(job_template, post, admin_user, default, status): + job_template.survey_enabled = True + job_template.save() + input_data = { + 'description': 'A survey', + 'spec': [{ + 'index': 0, + 'question_name': 'What is your password?', + 'required': True, + 'variable': 'secret_value', + 'default': default, + 'type': 'password' + }], + 'name': 'my survey' + } + resp = post(url=reverse('api:job_template_survey_spec', kwargs={'pk': job_template.id}), + data=input_data, user=admin_user, expect=status) + + if status == 200: + updated_jt = JobTemplate.objects.get(pk=job_template.pk) + assert updated_jt.survey_spec['spec'][0]['default'].startswith('$encrypted$') + + job = updated_jt.create_unified_job() + assert json.loads(job.extra_vars)['secret_value'].startswith('$encrypted$') + assert json.loads(job.decrypted_extra_vars()) == { + 'secret_value': default + } + else: + assert "for 'secret_value' expected to be a string." in str(resp.data) + + +@mock.patch('awx.api.views.feature_enabled', lambda feature: True) +@pytest.mark.django_db +def test_survey_spec_default_passwords_encrypted_on_update(job_template, post, put, admin_user): + input_data = { + 'description': 'A survey', + 'spec': [{ + 'index': 0, + 'question_name': 'What is your password?', + 'required': True, + 'variable': 'secret_value', + 'default': 'SUPERSECRET', + 'type': 'password' + }], + 'name': 'my survey' + } + post(url=reverse('api:job_template_survey_spec', kwargs={'pk': job_template.id}), + data=input_data, user=admin_user, expect=200) + updated_jt = JobTemplate.objects.get(pk=job_template.pk) + + # simulate a survey field edit where we're not changing the default value + input_data['spec'][0]['default'] = '$encrypted$' + post(url=reverse('api:job_template_survey_spec', kwargs={'pk': job_template.id}), + data=input_data, user=admin_user, expect=200) + assert updated_jt.survey_spec == JobTemplate.objects.get(pk=job_template.pk).survey_spec + + # Tests related to survey content validation @mock.patch('awx.api.views.feature_enabled', lambda feature: True) @pytest.mark.django_db diff --git a/awx/main/tests/unit/models/test_survey_models.py b/awx/main/tests/unit/models/test_survey_models.py index ba05940abc..d40d6f4800 100644 --- a/awx/main/tests/unit/models/test_survey_models.py +++ b/awx/main/tests/unit/models/test_survey_models.py @@ -13,6 +13,7 @@ from awx.main.models import ( @pytest.fixture def job(mocker): ret = mocker.MagicMock(**{ + 'decrypted_extra_vars.return_value': '{\"secret_key\": \"my_password\"}', 'display_extra_vars.return_value': '{\"secret_key\": \"$encrypted$\"}', 'extra_vars_dict': {"secret_key": "my_password"}, 'pk': 1, 'job_template.pk': 1, 'job_template.name': '', diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index a9fb3dd448..ccd49a5094 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -31,7 +31,7 @@ from awx.main.models import ( ) from awx.main import tasks -from awx.main.utils import encrypt_field +from awx.main.utils import encrypt_field, encrypt_value @@ -306,6 +306,20 @@ class TestGenericRun(TestJobExecution): assert '"awx_user_id": 123,' in ' '.join(args) assert '"awx_user_name": "angry-spud"' in ' '.join(args) + def test_survey_extra_vars(self): + self.instance.extra_vars = json.dumps({ + 'super_secret': encrypt_value('CLASSIFIED', pk=None) + }) + self.instance.survey_passwords = { + 'super_secret': '$encrypted$' + } + self.task.run(self.pk) + + assert self.run_pexpect.call_count == 1 + call_args, _ = self.run_pexpect.call_args_list[0] + args, cwd, env, stdout = call_args + assert '"super_secret": "CLASSIFIED"' in ' '.join(args) + def test_awx_task_env(self): patch = mock.patch('awx.main.tasks.settings.AWX_TASK_ENV', {'FOO': 'BAR'}) patch.start() diff --git a/awx/main/utils/encryption.py b/awx/main/utils/encryption.py index 025081b773..c8c5b72afd 100644 --- a/awx/main/utils/encryption.py +++ b/awx/main/utils/encryption.py @@ -1,6 +1,7 @@ import base64 import hashlib import logging +from collections import namedtuple import six from cryptography.fernet import Fernet, InvalidToken @@ -8,7 +9,8 @@ from cryptography.hazmat.backends import default_backend from django.utils.encoding import smart_str -__all__ = ['get_encryption_key', 'encrypt_field', 'decrypt_field', 'decrypt_value'] +__all__ = ['get_encryption_key', 'encrypt_value', 'encrypt_field', + 'decrypt_field', 'decrypt_value'] logger = logging.getLogger('awx.main.utils.encryption') @@ -50,6 +52,11 @@ def get_encryption_key(field_name, pk=None): return base64.urlsafe_b64encode(h.digest()) +def encrypt_value(value, pk=None): + TransientField = namedtuple('TransientField', ['pk', 'value']) + return encrypt_field(TransientField(pk=pk, value=value), 'value') + + def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False): ''' Return content of the given instance and field name encrypted. From f26bdb3e966c3f649929663cd7066f464bd0146e Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 29 Sep 2017 11:41:39 -0400 Subject: [PATCH 017/141] migrate existing survey passwords to be encrypted see: https://github.com/ansible/ansible-tower/issues/7046 --- .../0011_v322_encrypt_survey_passwords.py | 17 ++++++ awx/main/migrations/_reencrypt.py | 37 ++++++++++++ .../functional/test_reencrypt_migration.py | 56 ++++++++++++++++++- 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 awx/main/migrations/0011_v322_encrypt_survey_passwords.py diff --git a/awx/main/migrations/0011_v322_encrypt_survey_passwords.py b/awx/main/migrations/0011_v322_encrypt_survey_passwords.py new file mode 100644 index 0000000000..344aa96bc1 --- /dev/null +++ b/awx/main/migrations/0011_v322_encrypt_survey_passwords.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +from awx.main.migrations import ActivityStreamDisabledMigration +from awx.main.migrations import _reencrypt as reencrypt + + +class Migration(ActivityStreamDisabledMigration): + + dependencies = [ + ('main', '0010_v322_add_support_for_ovirt4_inventory'), + ] + + operations = [ + migrations.RunPython(reencrypt.encrypt_survey_passwords), + ] diff --git a/awx/main/migrations/_reencrypt.py b/awx/main/migrations/_reencrypt.py index c4e502f9a2..c1fad4aedf 100644 --- a/awx/main/migrations/_reencrypt.py +++ b/awx/main/migrations/_reencrypt.py @@ -1,5 +1,7 @@ import logging +import json from django.utils.translation import ugettext_lazy as _ +import six from awx.conf.migrations._reencrypt import ( decrypt_field, @@ -72,3 +74,38 @@ def _unified_jobs(apps): uj.start_args = decrypt_field(uj, 'start_args') uj.start_args = encrypt_field(uj, 'start_args') uj.save() + + +def encrypt_survey_passwords(apps, schema_editor): + _encrypt_survey_passwords( + apps.get_model('main', 'Job'), + apps.get_model('main', 'JobTemplate'), + ) + + +def _encrypt_survey_passwords(Job, JobTemplate): + from awx.main.utils.encryption import encrypt_value + for jt in JobTemplate.objects.exclude(survey_spec={}): + changed = False + if jt.survey_spec.get('spec', []): + for field in jt.survey_spec['spec']: + if field.get('type') == 'password' and field.get('default', ''): + if field['default'].startswith('$encrypted$'): + continue + field['default'] = encrypt_value(field['default'], pk=None) + changed = True + if changed: + jt.save() + + for job in Job.objects.defer('result_stdout_text').exclude(survey_passwords={}).iterator(): + changed = False + for key in job.survey_passwords: + if key in job.extra_vars: + extra_vars = json.loads(job.extra_vars) + if not extra_vars.get(key, '') or extra_vars[key].startswith('$encrypted$'): + continue + extra_vars[key] = encrypt_value(extra_vars[key], pk=None) + job.extra_vars = json.dumps(extra_vars) + changed = True + if changed: + job.save() diff --git a/awx/main/tests/functional/test_reencrypt_migration.py b/awx/main/tests/functional/test_reencrypt_migration.py index 3201866893..18aa2bc644 100644 --- a/awx/main/tests/functional/test_reencrypt_migration.py +++ b/awx/main/tests/functional/test_reencrypt_migration.py @@ -10,6 +10,8 @@ from django.apps import apps from awx.main.models import ( UnifiedJob, + Job, + JobTemplate, NotificationTemplate, Credential, ) @@ -20,9 +22,10 @@ from awx.main.migrations._reencrypt import ( _notification_templates, _credentials, _unified_jobs, + _encrypt_survey_passwords ) -from awx.main.utils import decrypt_field +from awx.main.utils import decrypt_field, get_encryption_key, decrypt_value @pytest.mark.django_db @@ -93,3 +96,54 @@ def test_unified_job_migration(old_enc, new_enc, value): # Exception if the encryption type of AESCBC is not properly skipped, ensures # our `startswith` calls don't have typos _unified_jobs(apps) + + +@pytest.mark.django_db +def test_survey_default_password_encryption(job_template_factory): + jt = job_template_factory('jt', organization='org1', project='prj', + inventory='inv', credential='cred').job_template + jt.survey_enabled = True + jt.survey_spec = { + 'description': 'A survey', + 'spec': [{ + 'index': 0, + 'question_name': 'What is your password?', + 'required': True, + 'variable': 'secret_value', + 'default': 'SUPERSECRET', + 'type': 'password' + }], + 'name': 'my survey' + } + jt.save() + + _encrypt_survey_passwords(Job, JobTemplate) + spec = JobTemplate.objects.get(pk=jt.pk).survey_spec['spec'] + assert decrypt_value(get_encryption_key('value', pk=None), spec[0]['default']) == 'SUPERSECRET' + + +@pytest.mark.django_db +def test_job_survey_vars_encryption(job_template_factory): + jt = job_template_factory('jt', organization='org1', project='prj', + inventory='inv', credential='cred').job_template + jt.survey_enabled = True + jt.survey_spec = { + 'description': 'A survey', + 'spec': [{ + 'index': 0, + 'question_name': 'What is your password?', + 'required': True, + 'variable': 'secret_value', + 'default': '', + 'type': 'password' + }], + 'name': 'my survey' + } + jt.save() + job = jt.create_unified_job() + job.extra_vars = json.dumps({'secret_value': 'SUPERSECRET'}) + job.save() + + _encrypt_survey_passwords(Job, JobTemplate) + job = Job.objects.get(pk=job.pk) + assert json.loads(job.decrypted_extra_vars()) == {'secret_value': 'SUPERSECRET'} From 3abbe87e10ef8dfab4e612e330a67f5327170b23 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 2 Oct 2017 11:17:18 -0400 Subject: [PATCH 018/141] fix bug checking WFJT node for prompted resources --- awx/main/access.py | 4 ++-- awx/main/tests/functional/test_rbac_workflow.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index c2dae1397c..75fd3ce1e7 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1605,9 +1605,9 @@ class WorkflowJobTemplateNodeAccess(BaseAccess): if 'credential' in data or 'inventory' in data: new_data = data if 'credential' not in data: - new_data['credential'] = self.credential + new_data['credential'] = obj.credential if 'inventory' not in data: - new_data['inventory'] = self.inventory + new_data['inventory'] = obj.inventory return self.can_use_prompted_resources(new_data) return True diff --git a/awx/main/tests/functional/test_rbac_workflow.py b/awx/main/tests/functional/test_rbac_workflow.py index c069a8dbad..84b02b5690 100644 --- a/awx/main/tests/functional/test_rbac_workflow.py +++ b/awx/main/tests/functional/test_rbac_workflow.py @@ -57,7 +57,21 @@ class TestWorkflowJobTemplateNodeAccess: # without access to the related job template, admin to the WFJT can # not change the prompted parameters access = WorkflowJobTemplateNodeAccess(org_admin) - assert not access.can_change(wfjt_node, {'job_type': 'scan'}) + assert not access.can_change(wfjt_node, {'job_type': 'check'}) + + def test_node_edit_allowed(self, wfjt_node, org_admin): + wfjt_node.unified_job_template.admin_role.members.add(org_admin) + access = WorkflowJobTemplateNodeAccess(org_admin) + assert access.can_change(wfjt_node, {'job_type': 'check'}) + + def test_prompted_resource_prevents_edit(self, wfjt_node, org_admin, machine_credential): + # without access to prompted resources, admin to the WFJT can + # not change the other prompted resources + wfjt_node.unified_job_template.admin_role.members.add(org_admin) + wfjt_node.credential = machine_credential + wfjt_node.save() + access = WorkflowJobTemplateNodeAccess(org_admin) + assert not access.can_change(wfjt_node, {'inventory': 45}) def test_add_JT_no_start_perm(self, wfjt, job_template, rando): wfjt.admin_role.members.add(rando) From c45fbcf2eeda2b51231c870cc9d27343c768988f Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 28 Sep 2017 12:01:58 -0400 Subject: [PATCH 019/141] add IG committed capacity to serializer --- awx/api/serializers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index fbc9db29f5..02dc7796fd 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -3663,6 +3663,7 @@ class InstanceSerializer(BaseSerializer): class InstanceGroupSerializer(BaseSerializer): + committed_capacity = serializers.SerializerMethodField() consumed_capacity = serializers.SerializerMethodField() percent_capacity_remaining = serializers.SerializerMethodField() jobs_running = serializers.SerializerMethodField() @@ -3670,7 +3671,8 @@ class InstanceGroupSerializer(BaseSerializer): class Meta: model = InstanceGroup - fields = ("id", "type", "url", "related", "name", "created", "modified", "capacity", "consumed_capacity", + fields = ("id", "type", "url", "related", "name", "created", "modified", + "capacity", "committed_capacity", "consumed_capacity", "percent_capacity_remaining", "jobs_running", "instances", "controller") def get_related(self, obj): @@ -3699,7 +3701,10 @@ class InstanceGroupSerializer(BaseSerializer): return self.context['capacity_map'] def get_consumed_capacity(self, obj): - return self.get_capacity_dict()[obj.name]['consumed_capacity'] + return self.get_capacity_dict()[obj.name]['running_capacity'] + + def get_committed_capacity(self, obj): + return self.get_capacity_dict()[obj.name]['committed_capacity'] def get_percent_capacity_remaining(self, obj): if not obj.capacity: From b372cebf8d01c28aa7f01d8d98d6b438134dd22c Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 26 Jun 2017 15:07:23 -0400 Subject: [PATCH 020/141] fix a bug when Tower is integrated with ipsilon SAML server https://github.com/ansible/ansible-tower/issues/6683 --- awx/sso/backends.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/awx/sso/backends.py b/awx/sso/backends.py index 5ed0385018..aa62311007 100644 --- a/awx/sso/backends.py +++ b/awx/sso/backends.py @@ -240,7 +240,10 @@ class TowerSAMLIdentityProvider(BaseSAMLIdentityProvider): another attribute to use. """ key = self.conf.get(conf_key, default_attribute) - value = attributes[key][0] if key in attributes else None + value = attributes[key] if key in attributes else None + # In certain implementations (like https://pagure.io/ipsilon) this value is a string, not a list + if isinstance(value, (list, tuple)): + value = value[0] if conf_key in ('attr_first_name', 'attr_last_name', 'attr_username', 'attr_email') and value is None: logger.warn("Could not map user detail '%s' from SAML attribute '%s'; " "update SOCIAL_AUTH_SAML_ENABLED_IDPS['%s']['%s'] with the correct SAML attribute.", From 82d05e0a1031a89805acb9083870d224b096272f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 4 Oct 2017 16:03:45 -0400 Subject: [PATCH 021/141] properly sanitize encrypted default passwords in JT.survey_spec see: https://github.com/ansible/ansible-tower/issues/7259 --- awx/api/views.py | 7 +------ awx/main/models/mixins.py | 11 +++++++++++ awx/main/tests/unit/api/test_views.py | 14 -------------- awx/main/tests/unit/models/test_survey_models.py | 9 +++++++++ 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 366ee29957..00a862dd23 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2851,13 +2851,8 @@ class JobTemplateSurveySpec(GenericAPIView): if not feature_enabled('surveys'): raise LicenseForbids(_('Your license does not allow ' 'adding surveys.')) - survey_spec = obj.survey_spec - for pos, field in enumerate(survey_spec.get('spec', [])): - if field.get('type') == 'password': - if 'default' in field and field['default']: - field['default'] = '$encrypted$' - return Response(survey_spec) + return Response(obj.display_survey_spec()) def post(self, request, *args, **kwargs): obj = self.get_object() diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 9bdf7194d7..1186797b09 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -240,6 +240,17 @@ class SurveyJobTemplateMixin(models.Model): errors += self._survey_element_validation(survey_element, data) return errors + def display_survey_spec(self): + ''' + Hide encrypted default passwords in survey specs + ''' + survey_spec = self.survey_spec.copy() if self.survey_spec else {} + for field in survey_spec.get('spec', []): + if field.get('type') == 'password': + if 'default' in field and field['default']: + field['default'] = '$encrypted$' + return survey_spec + class SurveyJobMixin(models.Model): class Meta: diff --git a/awx/main/tests/unit/api/test_views.py b/awx/main/tests/unit/api/test_views.py index a2532a75d7..68ba754d48 100644 --- a/awx/main/tests/unit/api/test_views.py +++ b/awx/main/tests/unit/api/test_views.py @@ -7,7 +7,6 @@ from collections import namedtuple from awx.api.views import ( ApiVersionRootView, JobTemplateLabelList, - JobTemplateSurveySpec, InventoryInventorySourcesUpdate, InventoryHostsList, HostInsights, @@ -80,19 +79,6 @@ class TestJobTemplateLabelList: assert mixin_unattach.called_with(mock_request, None, None) -class TestJobTemplateSurveySpec(object): - @mock.patch('awx.api.views.feature_enabled', lambda feature: True) - def test_get_password_type(self, mocker, mock_response_new): - JobTemplate = namedtuple('JobTemplate', 'survey_spec') - obj = JobTemplate(survey_spec={'spec':[{'type': 'password', 'default': 'my_default'}]}) - with mocker.patch.object(JobTemplateSurveySpec, 'get_object', return_value=obj): - view = JobTemplateSurveySpec() - response = view.get(mocker.MagicMock()) - assert response == mock_response_new - # which there was a better way to do this! - assert response.call_args[0][1]['spec'][0]['default'] == '$encrypted$' - - class TestInventoryInventorySourcesUpdate: @pytest.mark.parametrize("can_update, can_access, is_source, is_up_on_proj, expected", [ diff --git a/awx/main/tests/unit/models/test_survey_models.py b/awx/main/tests/unit/models/test_survey_models.py index d40d6f4800..d283c57081 100644 --- a/awx/main/tests/unit/models/test_survey_models.py +++ b/awx/main/tests/unit/models/test_survey_models.py @@ -94,6 +94,15 @@ def test_update_kwargs_survey_invalid_default(survey_spec_factory): assert json.loads(defaulted_extra_vars['extra_vars'])['var2'] == 2 +@pytest.mark.survey +def test_display_survey_spec_encrypts_default(survey_spec_factory): + spec = survey_spec_factory('var2') + spec['spec'][0]['type'] = 'password' + spec['spec'][0]['default'] = 'some-default' + jt = JobTemplate(name="test-jt", survey_spec=spec, survey_enabled=True) + assert jt.display_survey_spec()['spec'][0]['default'] == '$encrypted$' + + @pytest.mark.survey @pytest.mark.parametrize("question_type,default,min,max,expect_use,expect_value", [ ("text", "", 0, 0, True, ''), # default used From cc8b115c6ab8265e5122e992a8ebe9960c92ada9 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Mon, 2 Oct 2017 10:59:52 -0400 Subject: [PATCH 022/141] Fix SAML auth behind load balancer issue. Relates to #7586 of ansible-tower as a follow-up of fix #420 of tower. The original fix works for Django version 1.9 and above, this PR expanded the solution to Django verison 1.8 and below. Signed-off-by: Aaron Tan --- awx/sso/strategies/django_strategy.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/awx/sso/strategies/django_strategy.py b/awx/sso/strategies/django_strategy.py index 74fcb94117..0c8c1b492d 100644 --- a/awx/sso/strategies/django_strategy.py +++ b/awx/sso/strategies/django_strategy.py @@ -1,6 +1,10 @@ # Copyright (c) 2017 Ansible, Inc. # All Rights Reserved. +# Django +from django.conf import settings + +# Python social auth from social.strategies.django_strategy import DjangoStrategy @@ -9,8 +13,9 @@ class AWXDjangoStrategy(DjangoStrategy): fixes and updates from social-app-django TODO: Revert back to using the default DjangoStrategy after - we upgrade to social-core / social-app-django. We will also - want to ensure we update the SOCIAL_AUTH_STRATEGY setting. + we upgrade to social-core / social-app-django and upgrade Django + to 1.9 and above. We will also want to ensure we update the + SOCIAL_AUTH_STRATEGY setting. """ def request_port(self): @@ -24,4 +29,7 @@ class AWXDjangoStrategy(DjangoStrategy): try: return host_parts[1] except IndexError: - return self.request.META['SERVER_PORT'] + if settings.USE_X_FORWARDED_PORT and 'HTTP_X_FORWARDED_PORT' in self.request.META: + return self.request.META['HTTP_X_FORWARDED_PORT'] + else: + return self.request.META['SERVER_PORT'] From d2e0b2628726ba6bafc2c647bb28ae535e5ff0f7 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 2 Oct 2017 13:48:48 -0400 Subject: [PATCH 023/141] allow deleting hosts and groups from inv src sublists --- awx/api/generics.py | 53 ++++++++++++++----- .../api/sub_list_destroy_api_view.md | 6 +++ awx/api/views.py | 6 ++- awx/main/tests/functional/api/test_generic.py | 31 +++++++++++ awx/main/tests/functional/conftest.py | 2 +- 5 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 awx/api/templates/api/sub_list_destroy_api_view.md diff --git a/awx/api/generics.py b/awx/api/generics.py index 36176016b0..fcc48322ce 100644 --- a/awx/api/generics.py +++ b/awx/api/generics.py @@ -37,9 +37,10 @@ from awx.api.metadata import SublistAttachDetatchMetadata __all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView', 'ListCreateAPIView', 'SubListAPIView', 'SubListCreateAPIView', + 'SubListDestroyAPIView', 'SubListCreateAttachDetachAPIView', 'RetrieveAPIView', 'RetrieveUpdateAPIView', 'RetrieveDestroyAPIView', - 'RetrieveUpdateDestroyAPIView', 'DestroyAPIView', + 'RetrieveUpdateDestroyAPIView', 'SubDetailAPIView', 'ResourceAccessList', 'ParentMixin', @@ -440,6 +441,41 @@ class SubListAPIView(ParentMixin, ListAPIView): return qs & sublist_qs +class DestroyAPIView(generics.DestroyAPIView): + + def has_delete_permission(self, obj): + return self.request.user.can_access(self.model, 'delete', obj) + + def perform_destroy(self, instance, check_permission=True): + if check_permission and not self.has_delete_permission(instance): + raise PermissionDenied() + super(DestroyAPIView, self).perform_destroy(instance) + + +class SubListDestroyAPIView(DestroyAPIView, SubListAPIView): + """ + Concrete view for deleting everything related by `relationship`. + """ + check_sub_obj_permission = True + + def destroy(self, request, *args, **kwargs): + instance_list = self.get_queryset() + if (not self.check_sub_obj_permission and + not request.user.can_access(self.parent_model, 'delete', self.get_parent_object())): + raise PermissionDenied() + self.perform_list_destroy(instance_list) + return Response(status=status.HTTP_204_NO_CONTENT) + + def perform_list_destroy(self, instance_list): + if self.check_sub_obj_permission: + # Check permissions for all before deleting, avoiding half-deleted lists + for instance in instance_list: + if self.has_delete_permission(instance): + raise PermissionDenied() + for instance in instance_list: + self.perform_destroy(instance, check_permission=False) + + class SubListCreateAPIView(SubListAPIView, ListCreateAPIView): # Base class for a sublist view that allows for creating subobjects # associated with the parent object. @@ -678,22 +714,11 @@ class RetrieveUpdateAPIView(RetrieveAPIView, generics.RetrieveUpdateAPIView): pass -class RetrieveDestroyAPIView(RetrieveAPIView, generics.RetrieveDestroyAPIView): - - def destroy(self, request, *args, **kwargs): - # somewhat lame that delete has to call it's own permissions check - obj = self.get_object() - if not request.user.can_access(self.model, 'delete', obj): - raise PermissionDenied() - obj.delete() - return Response(status=status.HTTP_204_NO_CONTENT) - - -class RetrieveUpdateDestroyAPIView(RetrieveUpdateAPIView, RetrieveDestroyAPIView): +class RetrieveDestroyAPIView(RetrieveAPIView, DestroyAPIView): pass -class DestroyAPIView(GenericAPIView, generics.DestroyAPIView): +class RetrieveUpdateDestroyAPIView(RetrieveUpdateAPIView, DestroyAPIView): pass diff --git a/awx/api/templates/api/sub_list_destroy_api_view.md b/awx/api/templates/api/sub_list_destroy_api_view.md new file mode 100644 index 0000000000..72654f88a0 --- /dev/null +++ b/awx/api/templates/api/sub_list_destroy_api_view.md @@ -0,0 +1,6 @@ +{% include "api/sub_list_create_api_view.md" %} + +# Delete all {{ model_verbose_name_plural }} of this {{ parent_model_verbose_name|title }}: + +Make a DELETE request to this resource to delete all {{ model_verbose_name_plural }} show in the list. +The {{ parent_model_verbose_name|title }} will not be deleted by this request. diff --git a/awx/api/views.py b/awx/api/views.py index 366ee29957..9763185c30 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2616,23 +2616,25 @@ class InventorySourceNotificationTemplatesSuccessList(InventorySourceNotificatio relationship = 'notification_templates_success' -class InventorySourceHostsList(SubListAPIView): +class InventorySourceHostsList(SubListDestroyAPIView): model = Host serializer_class = HostSerializer parent_model = InventorySource relationship = 'hosts' new_in_148 = True + check_sub_obj_permission = False capabilities_prefetch = ['inventory.admin'] -class InventorySourceGroupsList(SubListAPIView): +class InventorySourceGroupsList(SubListDestroyAPIView): model = Group serializer_class = GroupSerializer parent_model = InventorySource relationship = 'groups' new_in_148 = True + check_sub_obj_permission = False class InventorySourceUpdatesList(SubListAPIView): diff --git a/awx/main/tests/functional/api/test_generic.py b/awx/main/tests/functional/api/test_generic.py index 6b2d580d2d..4d68b43ead 100644 --- a/awx/main/tests/functional/api/test_generic.py +++ b/awx/main/tests/functional/api/test_generic.py @@ -60,3 +60,34 @@ def test_proxy_ip_whitelist(get, patch, admin): REMOTE_HOST='my.proxy.example.org', HTTP_X_FROM_THE_LOAD_BALANCER='some-actual-ip') assert middleware.environ['HTTP_X_FROM_THE_LOAD_BALANCER'] == 'some-actual-ip' + + +@pytest.mark.django_db +class TestDeleteViews: + def test_sublist_delete_permission_check(self, inventory_source, host, rando, delete): + inventory_source.hosts.add(host) + inventory_source.inventory.read_role.members.add(rando) + delete( + reverse( + 'api:inventory_source_hosts_list', + kwargs={'version': 'v2', 'pk': inventory_source.pk} + ), user=rando, expect=403 + ) + + def test_sublist_delete_functionality(self, inventory_source, host, rando, delete): + inventory_source.hosts.add(host) + inventory_source.inventory.admin_role.members.add(rando) + delete( + reverse( + 'api:inventory_source_hosts_list', + kwargs={'version': 'v2', 'pk': inventory_source.pk} + ), user=rando, expect=204 + ) + assert inventory_source.hosts.count() == 0 + + def test_destroy_permission_check(self, job_factory, system_auditor, delete): + job = job_factory() + resp = delete( + job.get_absolute_url(), user=system_auditor + ) + assert resp.status_code == 403 diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index 456d1a409e..2a4fea237f 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -377,7 +377,7 @@ def admin(user): @pytest.fixture def system_auditor(user): - u = user(False) + u = user('sys-auditor', False) Role.singleton('system_auditor').members.add(u) return u From 358ef76529d2f88c53ac9758151d2762d37c7d0d Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Fri, 22 Sep 2017 16:06:59 -0400 Subject: [PATCH 024/141] Remove search term separators Relates #7656 in ansible-tower. We have been using comma `,` and space ` ` to separate search terms in query string `__search=`, however in general we can always use `&` to achieve separation like `__search=&__search=&...`. Using specific delimiters makes it impossible for search terms to contain those delimiters, so they are better off being removed. Signed-off-by: Aaron Tan --- awx/api/filters.py | 9 ++++----- awx/api/templates/api/_list_common.md | 12 ++++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/awx/api/filters.py b/awx/api/filters.py index a231c8af8d..105403bd17 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -242,11 +242,10 @@ class FieldLookupBackend(BaseFilterBackend): # Search across related objects. if key.endswith('__search'): for value in values: - for search_term in force_text(value).replace(',', ' ').split(): - search_value, new_keys = self.value_to_python(queryset.model, key, search_term) - assert isinstance(new_keys, list) - for new_key in new_keys: - search_filters.append((new_key, search_value)) + search_value, new_keys = self.value_to_python(queryset.model, key, force_text(value)) + assert isinstance(new_keys, list) + for new_key in new_keys: + search_filters.append((new_key, search_value)) continue # Custom chain__ and or__ filters, mutually exclusive (both can diff --git a/awx/api/templates/api/_list_common.md b/awx/api/templates/api/_list_common.md index 706ae732a5..de58292756 100644 --- a/awx/api/templates/api/_list_common.md +++ b/awx/api/templates/api/_list_common.md @@ -1,9 +1,9 @@ The resulting data structure contains: { - "count": 99, - "next": null, - "previous": null, + "count": 99, + "next": null, + "previous": null, "results": [ ... ] @@ -60,6 +60,10 @@ _Added in AWX 1.4_ ?related__search=findme +Note: If you want to provide more than one search terms, please use multiple +search fields with the same key, like `?related__search=foo&related__search=bar`, +All search terms with the same key will be ORed together. + ## Filtering Any additional query string parameters may be used to filter the list of @@ -70,7 +74,7 @@ in the specified value should be url-encoded. For example: ?field=value%20xyz Fields may also span relations, only for fields and relationships defined in -the database: +the database: ?other__field=value From a01f80db5b628f217b4531b83c69f10e1365a085 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 6 Oct 2017 13:06:07 -0400 Subject: [PATCH 025/141] Exclude credential type content from v1 credential_type_id was showing up in vault_credential summary_fields in API v1 --- awx/api/serializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 02dc7796fd..1805f38352 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -343,7 +343,9 @@ class BaseSerializer(serializers.ModelSerializer): continue summary_fields[fk] = OrderedDict() for field in related_fields: - if field == 'credential_type_id' and fk == 'credential' and self.version < 2: # TODO: remove version check in 3.3 + if ( + self.version < 2 and field == 'credential_type_id' and + fk in ['credential', 'vault_credential']): # TODO: remove version check in 3.3 continue fval = getattr(fkval, field, None) From 5380d57ce88a7451d1c617852196ff8d6d19d35b Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 6 Oct 2017 13:23:25 -0400 Subject: [PATCH 026/141] add period to active job conflict error Rename StateConflict to ActiveJobConflict and used shared message inside of that exception class. --- awx/main/access.py | 31 +++++++++---------- .../tests/functional/test_rbac_inventory.py | 4 +-- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index 75fd3ce1e7..26675330d5 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -26,7 +26,7 @@ from awx.conf.license import LicenseForbids, feature_enabled __all__ = ['get_user_queryset', 'check_user_access', 'check_user_access_with_errors', 'user_accessible_objects', 'consumer_access', - 'user_admin_role', 'StateConflict',] + 'user_admin_role', 'ActiveJobConflict',] logger = logging.getLogger('awx.main.access') @@ -36,9 +36,15 @@ access_registry = { } -class StateConflict(ValidationError): +class ActiveJobConflict(ValidationError): status_code = 409 + def __init__(self, active_jobs): + super(ActiveJobConflict, self).__init__({ + "conflict": _("Resource is being used by running jobs."), + "active_jobs": active_jobs + }) + def register_access(model_class, access_class): access_registry[model_class] = access_class @@ -550,8 +556,7 @@ class OrganizationAccess(BaseAccess): active_jobs.extend([dict(type="inventory_update", id=o.id) for o in InventoryUpdate.objects.filter(inventory_source__inventory__organization=obj, status__in=ACTIVE_STATES)]) if len(active_jobs) > 0: - raise StateConflict({"conflict": _("Resource is being used by running jobs"), - "active_jobs": active_jobs}) + raise ActiveJobConflict(active_jobs) return True def can_attach(self, obj, sub_obj, relationship, *args, **kwargs): @@ -644,8 +649,7 @@ class InventoryAccess(BaseAccess): active_jobs.extend([dict(type="ad_hoc_command", id=o.id) for o in AdHocCommand.objects.filter(inventory=obj, status__in=ACTIVE_STATES)]) if len(active_jobs) > 0: - raise StateConflict({"conflict": _("Resource is being used by running jobs"), - "active_jobs": active_jobs}) + raise ActiveJobConflict(active_jobs) return True def can_run_ad_hoc_commands(self, obj): @@ -773,8 +777,7 @@ class GroupAccess(BaseAccess): active_jobs.extend([dict(type="inventory_update", id=o.id) for o in InventoryUpdate.objects.filter(inventory_source__in=obj.inventory_sources.all(), status__in=ACTIVE_STATES)]) if len(active_jobs) > 0: - raise StateConflict({"conflict": _("Resource is being used by running jobs"), - "active_jobs": active_jobs}) + raise ActiveJobConflict(active_jobs) return True def can_start(self, obj, validate_license=True): @@ -826,8 +829,7 @@ class InventorySourceAccess(BaseAccess): return False active_jobs_qs = InventoryUpdate.objects.filter(inventory_source=obj, status__in=ACTIVE_STATES) if active_jobs_qs.exists(): - raise StateConflict({"conflict": _("Resource is being used by running jobs"), - "active_jobs": [dict(type="inventory_update", id=o.id) for o in active_jobs_qs.all()]}) + raise ActiveJobConflict([dict(type="inventory_update", id=o.id) for o in active_jobs_qs.all()]) return True @check_superuser @@ -1092,8 +1094,7 @@ class ProjectAccess(BaseAccess): active_jobs.extend([dict(type="project_update", id=o.id) for o in ProjectUpdate.objects.filter(project=obj, status__in=ACTIVE_STATES)]) if len(active_jobs) > 0: - raise StateConflict({"conflict": _("Resource is being used by running jobs"), - "active_jobs": active_jobs}) + raise ActiveJobConflict(active_jobs) return True @check_superuser @@ -1298,8 +1299,7 @@ class JobTemplateAccess(BaseAccess): active_jobs = [dict(type="job", id=o.id) for o in obj.jobs.filter(status__in=ACTIVE_STATES)] if len(active_jobs) > 0: - raise StateConflict({"conflict": _("Resource is being used by running jobs"), - "active_jobs": active_jobs}) + raise ActiveJobConflict(active_jobs) return True @check_superuser @@ -1764,8 +1764,7 @@ class WorkflowJobTemplateAccess(BaseAccess): active_jobs = [dict(type="workflow_job", id=o.id) for o in obj.workflow_jobs.filter(status__in=ACTIVE_STATES)] if len(active_jobs) > 0: - raise StateConflict({"conflict": _("Resource is being used by running jobs"), - "active_jobs": active_jobs}) + raise ActiveJobConflict(active_jobs) return True diff --git a/awx/main/tests/functional/test_rbac_inventory.py b/awx/main/tests/functional/test_rbac_inventory.py index 821d8893a8..830e5a7b52 100644 --- a/awx/main/tests/functional/test_rbac_inventory.py +++ b/awx/main/tests/functional/test_rbac_inventory.py @@ -13,7 +13,7 @@ from awx.main.access import ( InventoryUpdateAccess, CustomInventoryScriptAccess, ScheduleAccess, - StateConflict + ActiveJobConflict ) @@ -21,7 +21,7 @@ from awx.main.access import ( def test_running_job_protection(inventory, admin_user): AdHocCommand.objects.create(inventory=inventory, status='running') access = InventoryAccess(admin_user) - with pytest.raises(StateConflict): + with pytest.raises(ActiveJobConflict): access.can_delete(inventory) From 2cc9e2ca0bec3074c300c2f1ed8ca97f7f57b6bb Mon Sep 17 00:00:00 2001 From: Marliana Lara Date: Mon, 2 Oct 2017 17:21:44 -0400 Subject: [PATCH 027/141] Fix hidden right border of form input lookup buttons --- awx/ui/client/legacy-styles/forms.less | 2 -- 1 file changed, 2 deletions(-) diff --git a/awx/ui/client/legacy-styles/forms.less b/awx/ui/client/legacy-styles/forms.less index 4fb40e395d..9508b254c6 100644 --- a/awx/ui/client/legacy-styles/forms.less +++ b/awx/ui/client/legacy-styles/forms.less @@ -458,7 +458,6 @@ align-items: center; justify-content: center; border:1px solid @field-border; - border-right: 0px; } .Form-lookupButton:hover { @@ -470,7 +469,6 @@ .Form-lookupButton:active, .Form-lookupButton:focus { border: 1px solid @field-border; - border-right: 0px; } .CodeMirror { From 1c374fba7d07eb506404cae7483f97a8c02e3ffe Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 6 Oct 2017 15:33:20 -0400 Subject: [PATCH 028/141] reword error message about encrypted user input --- awx/api/views.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 9157901b3b..7574b160a1 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2900,16 +2900,19 @@ class JobTemplateSurveySpec(GenericAPIView): if survey_item["type"] == "password" and "default" in survey_item: if not isinstance(survey_item['default'], six.string_types): - return Response( - _("Value %s for '%s' expected to be a string." % ( - survey_item["default"], survey_item["variable"] - )), - status=status.HTTP_400_BAD_REQUEST - ) + return Response(dict(error=_( + "Value {question_default} for '{variable_name}' expected to be a string." + ).format( + question_default=survey_item["default"], variable_name=survey_item["variable"]) + ), status=status.HTTP_400_BAD_REQUEST) elif survey_item["default"].startswith('$encrypted$'): if not obj.survey_spec: - return Response(dict(error=_("$encrypted$ is reserved keyword and may not be used as a default for password {}.".format(str(idx)))), - status=status.HTTP_400_BAD_REQUEST) + return Response(dict(error=_( + "$encrypted$ is reserved keyword for password questions and may not " + "be used as a default for '{variable_name}' in survey question {question_position}." + ).format( + variable_name=survey_item["variable"], question_position=str(idx)) + ), status=status.HTTP_400_BAD_REQUEST) else: old_spec = obj.survey_spec for old_item in old_spec['spec']: From dee4b7230386d04ea0414b9d9470e0a7f94c3bde Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 6 Oct 2017 16:16:17 -0400 Subject: [PATCH 029/141] add CTiT setting for max UI job events --- awx/api/views.py | 2 +- awx/settings/defaults.py | 2 +- awx/ui/conf.py | 11 +++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 9157901b3b..ceb3ed8389 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -3989,7 +3989,7 @@ class BaseJobEventsList(SubListAPIView): search_fields = ('stdout',) def finalize_response(self, request, response, *args, **kwargs): - response['X-UI-Max-Events'] = settings.RECOMMENDED_MAX_EVENTS_DISPLAY_HEADER + response['X-UI-Max-Events'] = settings.MAX_UI_JOB_EVENTS return super(BaseJobEventsList, self).finalize_response(request, response, *args, **kwargs) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 9f25ddb525..b64656fdbc 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -166,7 +166,7 @@ STDOUT_MAX_BYTES_DISPLAY = 1048576 # Returned in the header on event api lists as a recommendation to the UI # on how many events to display before truncating/hiding -RECOMMENDED_MAX_EVENTS_DISPLAY_HEADER = 4000 +MAX_UI_JOB_EVENTS = 4000 # The maximum size of the ansible callback event's res data structure # beyond this limit and the value will be removed diff --git a/awx/ui/conf.py b/awx/ui/conf.py index 8a8677cdf1..0d626c28f0 100644 --- a/awx/ui/conf.py +++ b/awx/ui/conf.py @@ -52,3 +52,14 @@ register( category_slug='ui', feature_required='rebranding', ) + +register( + 'MAX_UI_JOB_EVENTS', + field_class=fields.IntegerField, + min_value=100, + label=_('Max Job Events Retreived by UI'), + help_text=_('Maximum number of job events for the UI to retreive within a ' + 'single request.'), + category=_('UI'), + category_slug='ui', +) From b7071a48c2ab9c339bccbe1da21655f8a7a1322a Mon Sep 17 00:00:00 2001 From: gconsidine Date: Mon, 9 Oct 2017 11:06:55 -0400 Subject: [PATCH 030/141] Set lookup value changed from something to nothing to be null --- awx/ui/client/lib/components/form/form.directive.js | 4 ++-- awx/ui/client/lib/components/input/lookup.directive.js | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/awx/ui/client/lib/components/form/form.directive.js b/awx/ui/client/lib/components/form/form.directive.js index e4956d1101..4e04dd31f0 100644 --- a/awx/ui/client/lib/components/form/form.directive.js +++ b/awx/ui/client/lib/components/form/form.directive.js @@ -32,7 +32,7 @@ function AtFormController (eventService, strings) { vm.setListeners(); }; - vm.register = (category, component, el) => { + vm.register = (category, component, el) => { component.category = category; component.form = vm.state; @@ -66,7 +66,7 @@ function AtFormController (eventService, strings) { let data = vm.components .filter(component => component.category === 'input') .reduce((values, component) => { - if (!component.state._value) { + if (component.state._value === undefined) { return values; } diff --git a/awx/ui/client/lib/components/input/lookup.directive.js b/awx/ui/client/lib/components/input/lookup.directive.js index b2c60e15bd..9d6d9089d6 100644 --- a/awx/ui/client/lib/components/input/lookup.directive.js +++ b/awx/ui/client/lib/components/input/lookup.directive.js @@ -12,7 +12,7 @@ function atInputLookupLink (scope, element, attrs, controllers) { inputController.init(scope, element, formController); } -function AtInputLookupController (baseInputController, $q, $state, $stateParams) { +function AtInputLookupController (baseInputController, $q, $state) { let vm = this || {}; let scope; @@ -75,13 +75,14 @@ function AtInputLookupController (baseInputController, $q, $state, $stateParams) vm.resetDebounce = () => { clearTimeout(vm.debounce); - vm.searchAfterDebounce(); + vm.searchAfterDebounce(); }; vm.search = () => { scope.state._touched = true; if (scope.state._displayValue === '' && !scope.state._required) { + scope.state._value = null; return vm.check({ isValid: true }); } @@ -116,8 +117,7 @@ function AtInputLookupController (baseInputController, $q, $state, $stateParams) AtInputLookupController.$inject = [ 'BaseInputController', '$q', - '$state', - '$stateParams' + '$state' ]; function atInputLookup (pathService) { From c06778842885cd5a86bb15373b87015059a48f76 Mon Sep 17 00:00:00 2001 From: Chris Church Date: Sun, 8 Oct 2017 22:21:32 -0400 Subject: [PATCH 031/141] Support dash in LDAP attribute names in filters. --- awx/sso/tests/unit/test_ldap.py | 5 +++++ awx/sso/validators.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/awx/sso/tests/unit/test_ldap.py b/awx/sso/tests/unit/test_ldap.py index 48dd3e30e2..0a50871650 100644 --- a/awx/sso/tests/unit/test_ldap.py +++ b/awx/sso/tests/unit/test_ldap.py @@ -1,6 +1,7 @@ import ldap from awx.sso.backends import LDAPSettings +from awx.sso.validators import validate_ldap_filter def test_ldap_default_settings(mocker): @@ -19,3 +20,7 @@ def test_ldap_default_network_timeout(mocker): ldap.OPT_REFERRALS: 0, ldap.OPT_NETWORK_TIMEOUT: 30 } + + +def test_ldap_filter_validator(): + validate_ldap_filter('(test-uid=%(user)s)', with_user=True) diff --git a/awx/sso/validators.py b/awx/sso/validators.py index dd1086a426..7e89958236 100644 --- a/awx/sso/validators.py +++ b/awx/sso/validators.py @@ -47,7 +47,7 @@ def validate_ldap_filter(value, with_user=False): dn_value = value.replace('%(user)s', 'USER') else: dn_value = value - if re.match(r'^\([A-Za-z0-9]+?=[^()]+?\)$', dn_value): + if re.match(r'^\([A-Za-z0-9-]+?=[^()]+?\)$', dn_value): return elif re.match(r'^\([&|!]\(.*?\)\)$', dn_value): try: From 2fb67a36488cfe3aea37539ce08aeb54780525c0 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 9 Oct 2017 11:59:02 -0400 Subject: [PATCH 032/141] prevent OrderedDict syntax in error message --- awx/conf/fields.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/awx/conf/fields.py b/awx/conf/fields.py index b9b3503bb1..58f81b3d18 100644 --- a/awx/conf/fields.py +++ b/awx/conf/fields.py @@ -1,6 +1,7 @@ # Python import logging import urlparse +from collections import OrderedDict # Django from django.core.validators import URLValidator @@ -98,5 +99,7 @@ class KeyValueField(DictField): ret = super(KeyValueField, self).to_internal_value(data) for value in data.values(): if not isinstance(value, six.string_types + six.integer_types + (float,)): + if isinstance(value, OrderedDict): + value = dict(value) self.fail('invalid_child', input=value) return ret From 766a0887497b001ff9477fd9a1ba00062fb2312d Mon Sep 17 00:00:00 2001 From: gconsidine Date: Tue, 10 Oct 2017 10:54:05 -0400 Subject: [PATCH 033/141] Use credential_type to fetch associated types in list view --- .../credentials/legacy.credentials.js | 5 +--- awx/ui/client/lib/models/Base.js | 17 +++++++---- .../list/credentials-list.controller.js | 29 +++++++++++++++---- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/awx/ui/client/features/credentials/legacy.credentials.js b/awx/ui/client/features/credentials/legacy.credentials.js index e271da0936..4c2e2d1d81 100644 --- a/awx/ui/client/features/credentials/legacy.credentials.js +++ b/awx/ui/client/features/credentials/legacy.credentials.js @@ -41,10 +41,7 @@ function LegacyCredentialsService (pathService) { return qs.search(path, $stateParams[`${list.iterator}_search`]); } - ], - credentialType: ['CredentialTypeModel', CredentialType => { - return new CredentialType('get'); - }] + ] } }; diff --git a/awx/ui/client/lib/models/Base.js b/awx/ui/client/lib/models/Base.js index a1c60e1bd9..a8e7b3fb0b 100644 --- a/awx/ui/client/lib/models/Base.js +++ b/awx/ui/client/lib/models/Base.js @@ -43,21 +43,28 @@ function requestWithCache (method, resource) { * supported by the API. * * @arg {Object} params - An object of keys and values to to format and - * to the URL as a query string. Refer to the API documentation for the + * to the URL as a query string. Refer to the API documentation for the * resource in use for specifics. * @arg {Object} config - Configuration specific to the UI to accommodate * common use cases. * - * @yields {boolean} - Indicating a match has been found. If so, the results + * @yields {boolean} - Indicating a match has been found. If so, the results * are set on the model. */ -function search (params, config) { +function search (params = {}, config = {}) { let req = { method: 'GET', - url: this.path, - params + url: this.path }; + if (typeof params === 'string') { + req.url = `?params`; + } else if (Array.isArray(params)) { + req.url += `?${params.join('&')}`; + } else { + req.params = params; + } + return $http(req) .then(({ data }) => { if (!data.count) { diff --git a/awx/ui/client/src/credentials/list/credentials-list.controller.js b/awx/ui/client/src/credentials/list/credentials-list.controller.js index 924f58e367..ee623d8e13 100644 --- a/awx/ui/client/src/credentials/list/credentials-list.controller.js +++ b/awx/ui/client/src/credentials/list/credentials-list.controller.js @@ -5,10 +5,13 @@ *************************************************/ export default ['$scope', 'Rest', 'CredentialList', 'Prompt', 'ProcessErrors', 'GetBasePath', - 'Wait', '$state', '$filter', 'rbacUiControlService', 'Dataset', 'credentialType', 'i18n', + 'Wait', '$state', '$filter', 'rbacUiControlService', 'Dataset', 'CredentialTypeModel', + 'i18n', function($scope, Rest, CredentialList, Prompt, ProcessErrors, GetBasePath, Wait, $state, $filter, rbacUiControlService, Dataset, - credentialType, i18n) { + CredentialType, i18n) { + + const credentialType = new CredentialType(); var list = CredentialList, defaultUrl = GetBasePath('credentials'); @@ -45,9 +48,25 @@ export default ['$scope', 'Rest', 'CredentialList', 'Prompt', 'ProcessErrors', ' return; } - $scope[list.name].forEach(credential => { - credential.kind = credentialType.match('id', credential.credential_type).name; - }); + const params = $scope[list.name] + .reduce((accumulator, credential) => { + accumulator.push(credential.credential_type); + + return accumulator; + }, []) + .filter((id, i, array) => array.indexOf(id) === i) + .map(id => `or__id=${id}`); + + credentialType.search(params) + .then(found => { + if (!found) { + return; + } + + $scope[list.name].forEach(credential => { + credential.kind = credentialType.match('id', credential.credential_type).name; + }); + }); } // iterate over the list and add fields like type label, after the From b9483c28b0640a5dc93720e83382c5367ed6d257 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Tue, 10 Oct 2017 11:30:16 -0400 Subject: [PATCH 034/141] Disable inventory var overwrite in inv import Relates #7726 of ansible-tower. Signed-off-by: Aaron Tan --- .../management/commands/inventory_import.py | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 797355bf3b..f4ee1401fd 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -603,27 +603,20 @@ class Command(NoArgsCommand): def _update_inventory(self): ''' - Update/overwrite variables from "all" group. If importing from a - cloud source attached to a specific group, variables will be set on - the base group, otherwise they will be set on the whole inventory. + Update inventory variables from "all" group. ''' - # FIXME: figure out how "all" variables are handled in the new inventory source system + # TODO: We disable variable overwrite here in case user-defined inventory variables get + # mangled. But we still need to figure out a better way of processing multiple inventory + # update variables mixing with each other. all_obj = self.inventory - all_name = 'inventory' db_variables = all_obj.variables_dict - if self.overwrite_vars: - db_variables = self.all_group.variables - else: - db_variables.update(self.all_group.variables) + db_variables.update(self.all_group.variables) if db_variables != all_obj.variables_dict: all_obj.variables = json.dumps(db_variables) all_obj.save(update_fields=['variables']) - if self.overwrite_vars: - logger.info('%s variables replaced from "all" group', all_name.capitalize()) - else: - logger.info('%s variables updated from "all" group', all_name.capitalize()) + logger.info('Inventory variables updated from "all" group') else: - logger.info('%s variables unmodified', all_name.capitalize()) + logger.info('Inventory variables unmodified') def _create_update_groups(self): ''' From 341ef411a41b0134f403f7b9fbcff221931d6709 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 10 Oct 2017 13:16:25 -0400 Subject: [PATCH 035/141] update isolated container requirements --- tools/docker-isolated/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/docker-isolated/Dockerfile b/tools/docker-isolated/Dockerfile index b08a33e1ed..a7313e00b9 100644 --- a/tools/docker-isolated/Dockerfile +++ b/tools/docker-isolated/Dockerfile @@ -1,4 +1,4 @@ -FROM centos/systemd +FROM centos:7 RUN yum clean all ADD Makefile /tmp/Makefile @@ -6,7 +6,8 @@ RUN mkdir /tmp/requirements ADD requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt requirements/requirements_ansible_uninstall.txt requirements/requirements_isolated.txt /tmp/requirements/ RUN yum -y update && yum -y install curl epel-release RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - -RUN yum -y update && yum -y install openssh-server ansible mg vim tmux git python-devel python-psycopg2 make python-psutil libxml2-devel libxslt-devel libstdc++.so.6 gcc cyrus-sasl-devel cyrus-sasl openldap-devel libffi-devel zeromq-devel python-pip xmlsec1-devel swig krb5-devel xmlsec1-openssl xmlsec1 xmlsec1-openssl-devel libtool-ltdl-devel bubblewrap zanata-python-client gettext gcc-c++ +RUN yum -y localinstall http://download.postgresql.org/pub/repos/yum/9.4/redhat/rhel-6-x86_64/pgdg-centos94-9.4-3.noarch.rpm +RUN yum -y update && yum -y install openssh-server ansible mg vim tmux git python-devel python-psycopg2 make python-psutil libxml2-devel libxslt-devel libstdc++.so.6 gcc cyrus-sasl-devel cyrus-sasl openldap-devel libffi-devel zeromq-devel python-pip xmlsec1-devel swig krb5-devel xmlsec1-openssl xmlsec1 xmlsec1-openssl-devel libtool-ltdl-devel bubblewrap zanata-python-client gettext gcc-c++ libcurl-devel python-pycurl bzip2 RUN pip install virtualenv WORKDIR /tmp RUN make requirements_ansible From 03e58523b23ab757eab3ec6cdc2dab5018569ee7 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 6 Oct 2017 12:43:34 -0400 Subject: [PATCH 036/141] tweak of error message for ForeignKey filters --- awx/api/filters.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/awx/api/filters.py b/awx/api/filters.py index 105403bd17..2cf28c7c23 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -165,7 +165,13 @@ class FieldLookupBackend(BaseFilterBackend): elif isinstance(field, models.BooleanField): return to_python_boolean(value) elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)): - return self.to_python_related(value) + try: + return self.to_python_related(value) + except ValueError: + raise ParseError(_('Invalid {field_name} id: {field_id}').format( + field_name=getattr(field, 'name', 'related field'), + field_id=value) + ) else: return field.to_python(value) From e814f28039ce29db94301c0bc357a3939abfa268 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 6 Oct 2017 16:55:31 -0400 Subject: [PATCH 037/141] add logger statement for number of events --- awx/main/tasks.py | 6 ++++++ awx/main/utils/common.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 5dbadd4131..b27ce0e967 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -886,6 +886,12 @@ class BaseTask(LogErrorsTask): try: stdout_handle.flush() stdout_handle.close() + # If stdout_handle was wrapped with event filter, log data + if hasattr(stdout_handle, '_event_ct'): + logger.info('%s finished running, producing %s events.', + instance.log_format, stdout_handle._event_ct) + else: + logger.info('%s finished running', instance.log_format) except Exception: pass diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 22183c33dd..c370769f84 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -818,6 +818,7 @@ class OutputEventFilter(object): def __init__(self, fileobj=None, event_callback=None, raw_callback=None): self._fileobj = fileobj self._event_callback = event_callback + self._event_ct = 0 self._raw_callback = raw_callback self._counter = 1 self._start_line = 0 @@ -872,6 +873,7 @@ class OutputEventFilter(object): self._start_line += n_lines if self._event_callback: self._event_callback(event_data) + self._event_ct += 1 if next_event_data.get('uuid', None): self._current_event_data = next_event_data From 82160e2072576a6825a5eb51b414af95f4a34eaf Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Tue, 10 Oct 2017 13:26:31 -0400 Subject: [PATCH 038/141] Add LDAP deploy instructions Signed-off-by: Aaron Tan --- docs/auth/ldap.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/auth/ldap.md diff --git a/docs/auth/ldap.md b/docs/auth/ldap.md new file mode 100644 index 0000000000..b9a172c3d0 --- /dev/null +++ b/docs/auth/ldap.md @@ -0,0 +1,6 @@ +# LDAP +The Lightweight Directory Access Protocol (LDAP) is an open, vendor-neutral, industry standard application protocol for accessing and maintaining distributed directory information services over an Internet Protocol (IP) network. Directory services play an important role in developing intranet and Internet applications by allowing the sharing of information about users, systems, networks, services, and applications throughout the network. + +## Test environment setup + +Please see README.md of this repository: https://github.com/jangsutsr/deploy_ldap.git. From e66a1002ee6d20067e2a82f36c1361b484f2afe7 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 10 Oct 2017 14:58:09 -0400 Subject: [PATCH 039/141] fix equation for isolated instance capacity --- awx/plugins/isolated/awx_capacity.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/plugins/isolated/awx_capacity.py b/awx/plugins/isolated/awx_capacity.py index cc69d02a3b..cf370fb56e 100644 --- a/awx/plugins/isolated/awx_capacity.py +++ b/awx/plugins/isolated/awx_capacity.py @@ -41,7 +41,8 @@ def main(): total_mem_value = out.split()[7] if int(total_mem_value) <= 2048: cap = 50 - cap = 50 + ((int(total_mem_value) / 1024) - 2) * 75 + else: + cap = 50 + ((int(total_mem_value) / 1024) - 2) * 75 # Module never results in a change module.exit_json(changed=False, capacity=cap, version=version) From 9e3d90896b8316dd0385d18a9d6cb30c7617617e Mon Sep 17 00:00:00 2001 From: gconsidine Date: Tue, 10 Oct 2017 15:22:17 -0400 Subject: [PATCH 040/141] Remove unsupported tokens from search generated queries --- .../shared/smart-search/queryset.service.js | 73 ++++++++++--------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/awx/ui/client/src/shared/smart-search/queryset.service.js b/awx/ui/client/src/shared/smart-search/queryset.service.js index 1571ab936d..27681d18b4 100644 --- a/awx/ui/client/src/shared/smart-search/queryset.service.js +++ b/awx/ui/client/src/shared/smart-search/queryset.service.js @@ -29,43 +29,48 @@ export default ['$q', 'Rest', 'ProcessErrors', '$rootScope', 'Wait', 'DjangoSear return defer.promise; }, + replaceDefaultFlags (value) { + value = value.toString().replace(/__icontains_DEFAULT/g, "__icontains"); + value = value.toString().replace(/__search_DEFAULT/g, "__search"); + + return value; + }, + + replaceEncodedTokens(value) { + return decodeURIComponent(value).replace(/"|'/g, ""); + }, + + encodeTerms (values, key) { + key = this.replaceDefaultFlags(key); + + if (!Array.isArray(values)) { + values = this.replaceEncodedTokens(values); + + return `${key}=${values}`; + } + + return values + .map(value => { + value = this.replaceDefaultFlags(value); + value = this.replaceEncodedTokens(value); + + return `${key}=${value}`; + }) + .join('&'); + }, // encodes ui-router params from {operand__key__comparator: value} pairs to API-consumable URL encodeQueryset(params) { - let queryset; - queryset = _.reduce(params, (result, value, key) => { - return result + encodeTerm(value, key); - }, ''); - queryset = queryset.substring(0, queryset.length - 1); - return angular.isObject(params) ? `?${queryset}` : ''; - - function encodeTerm(value, key){ - - key = key.toString().replace(/__icontains_DEFAULT/g, "__icontains"); - key = key.toString().replace(/__search_DEFAULT/g, "__search"); - - value = value.toString().replace(/__icontains_DEFAULT/g, "__icontains"); - value = value.toString().replace(/__search_DEFAULT/g, "__search"); - - if (Array.isArray(value)){ - value = _.uniq(_.flattenDeep(value)); - let concated = ''; - angular.forEach(value, function(item){ - if(item && typeof item === 'string') { - item = decodeURIComponent(item).replace(/"|'/g, ""); - } - concated += `${key}=${item}&`; - }); - - return concated; - } - else { - if(value && typeof value === 'string') { - value = decodeURIComponent(value).replace(/"|'/g, ""); - } - - return `${key}=${value}&`; - } + if (typeof params !== 'object') { + return ''; } + + return _.reduce(params, (result, value, key) => { + if (result !== '?') { + result += '&'; + } + + return result += this.encodeTerms(value, key); + }, '?'); }, // encodes a ui smart-search param to a django-friendly param // operand:key:comparator:value => {operand__key__comparator: value} From eab82f3efaf5e173e974f645f3a4f38bcd2f0aaa Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 10 Oct 2017 15:45:20 -0400 Subject: [PATCH 041/141] updated fallback isolated version to 3.2.2 --- awx/main/expect/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/expect/run.py b/awx/main/expect/run.py index 3c248f9610..aed89f5f84 100755 --- a/awx/main/expect/run.py +++ b/awx/main/expect/run.py @@ -271,7 +271,7 @@ def __run__(private_data_dir): if __name__ == '__main__': - __version__ = '3.2.0' + __version__ = '3.2.2' try: import awx __version__ = awx.__version__ From 54640dbca0abaf766243dce8a05c9825bae57bb3 Mon Sep 17 00:00:00 2001 From: John Mitchell Date: Tue, 10 Oct 2017 17:08:04 -0400 Subject: [PATCH 042/141] hide workflow and survey buttons from non-detail tabs since the two are basically sub-states of the edit form (detail tab), they should only show up when that tab is selected --- awx/ui/client/src/templates/workflows.form.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/awx/ui/client/src/templates/workflows.form.js b/awx/ui/client/src/templates/workflows.form.js index 106d011cb6..ae147de530 100644 --- a/awx/ui/client/src/templates/workflows.form.js +++ b/awx/ui/client/src/templates/workflows.form.js @@ -163,7 +163,7 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n) { }, add_survey: { ngClick: 'addSurvey()', - ngShow: '!survey_exists && ($state.includes(\'templates.addWorkflowJobTemplate\') || $state.includes(\'templates.editWorkflowJobTemplate\'))', + ngShow: '!survey_exists && ($state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\'))', awFeature: 'surveys', awToolTip: '{{surveyTooltip}}', dataPlacement: 'top', @@ -173,7 +173,7 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n) { edit_survey: { ngClick: 'editSurvey()', awFeature: 'surveys', - ngShow: 'survey_exists && ($state.includes(\'templates.addWorkflowJobTemplate\') || $state.includes(\'templates.editWorkflowJobTemplate\'))', + ngShow: 'survey_exists && ($state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\'))', label: i18n._('Edit Survey'), class: 'Form-primaryButton', awToolTip: '{{surveyTooltip}}', @@ -181,7 +181,7 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n) { }, workflow_editor: { ngClick: 'openWorkflowMaker()', - ngShow: '$state.includes(\'templates.addWorkflowJobTemplate\') || $state.includes(\'templates.editWorkflowJobTemplate\')', + ngShow: '$state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\')', awToolTip: '{{workflowEditorTooltip}}', dataPlacement: 'top', label: i18n._('Workflow Editor'), From 1603106cb4733f45d97b40b4abc44274de087425 Mon Sep 17 00:00:00 2001 From: John Mitchell Date: Wed, 11 Oct 2017 10:58:38 -0400 Subject: [PATCH 043/141] include workflow editor when showing buttons' --- awx/ui/client/src/templates/workflows.form.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/ui/client/src/templates/workflows.form.js b/awx/ui/client/src/templates/workflows.form.js index ae147de530..d41cb7fb97 100644 --- a/awx/ui/client/src/templates/workflows.form.js +++ b/awx/ui/client/src/templates/workflows.form.js @@ -157,13 +157,13 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n) { view_survey: { ngClick: 'editSurvey()', awFeature: 'surveys', - ngShow: '($state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\')) && survey_exists && !(workflow_job_template_obj.summary_fields.user_capabilities.edit || canAddWorkflowJobTemplate)', + ngShow: '($state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate.workflowMaker\')) && survey_exists && !(workflow_job_template_obj.summary_fields.user_capabilities.edit || canAddWorkflowJobTemplate)', label: i18n._('View Survey'), class: 'Form-primaryButton' }, add_survey: { ngClick: 'addSurvey()', - ngShow: '!survey_exists && ($state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\'))', + ngShow: '$state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate.workflowMaker\')', awFeature: 'surveys', awToolTip: '{{surveyTooltip}}', dataPlacement: 'top', @@ -173,7 +173,7 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n) { edit_survey: { ngClick: 'editSurvey()', awFeature: 'surveys', - ngShow: 'survey_exists && ($state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\'))', + ngShow: '$state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate.workflowMaker\')', label: i18n._('Edit Survey'), class: 'Form-primaryButton', awToolTip: '{{surveyTooltip}}', @@ -181,7 +181,7 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n) { }, workflow_editor: { ngClick: 'openWorkflowMaker()', - ngShow: '$state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\')', + ngShow: '$state.is(\'templates.addWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate\') || $state.is(\'templates.editWorkflowJobTemplate.workflowMaker\')', awToolTip: '{{workflowEditorTooltip}}', dataPlacement: 'top', label: i18n._('Workflow Editor'), From 3ede367df42f9ce2bd489c43d956a0673f11f240 Mon Sep 17 00:00:00 2001 From: mabashian Date: Thu, 12 Oct 2017 16:52:59 -0400 Subject: [PATCH 044/141] Fixed smart inv button bug navigating to page 2 of hosts. Added tooltip when button is disabled. --- awx/ui/client/src/inventories-hosts/hosts/host.list.js | 6 ++++-- .../inventories-hosts/hosts/list/host-list.controller.js | 9 ++++++--- .../nested-hosts/group-nested-hosts-list.controller.js | 9 ++++++--- .../related/hosts/list/host-list.controller.js | 9 ++++++--- .../src/inventories-hosts/inventory-hosts.strings.js | 5 +++++ 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/awx/ui/client/src/inventories-hosts/hosts/host.list.js b/awx/ui/client/src/inventories-hosts/hosts/host.list.js index e91eebb30b..2422666878 100644 --- a/awx/ui/client/src/inventories-hosts/hosts/host.list.js +++ b/awx/ui/client/src/inventories-hosts/hosts/host.list.js @@ -104,12 +104,14 @@ export default ['i18n', function(i18n) { smart_inventory: { mode: 'all', ngClick: "smartInventory()", - awToolTip: i18n._("Create a new Smart Inventory from search results."), + awToolTip: "{{ smartInventoryButtonTooltip }}", + dataTipWatch: 'smartInventoryButtonTooltip', actionClass: 'btn List-buttonDefault', buttonContent: i18n._('SMART INVENTORY'), ngShow: 'canAdd && (hosts.length > 0 || !(searchTags | isEmpty))', dataPlacement: "top", - ngDisabled: '!enableSmartInventoryButton' + ngDisabled: '!enableSmartInventoryButton', + showTipWhenDisabled: true } } }; diff --git a/awx/ui/client/src/inventories-hosts/hosts/list/host-list.controller.js b/awx/ui/client/src/inventories-hosts/hosts/list/host-list.controller.js index 52a565b40c..693d403ea5 100644 --- a/awx/ui/client/src/inventories-hosts/hosts/list/host-list.controller.js +++ b/awx/ui/client/src/inventories-hosts/hosts/list/host-list.controller.js @@ -7,7 +7,7 @@ function HostsList($scope, HostsList, $rootScope, GetBasePath, rbacUiControlService, Dataset, $state, $filter, Prompt, Wait, - HostsService, SetStatus, canAdd) { + HostsService, SetStatus, canAdd, InventoryHostsStrings) { let list = HostsList; @@ -16,6 +16,7 @@ function HostsList($scope, HostsList, $rootScope, GetBasePath, function init(){ $scope.canAdd = canAdd; $scope.enableSmartInventoryButton = false; + $scope.smartInventoryButtonTooltip = InventoryHostsStrings.get('smartinventorybutton.DISABLED_INSTRUCTIONS'); // Search init $scope.list = list; @@ -37,14 +38,16 @@ function HostsList($scope, HostsList, $rootScope, GetBasePath, if(toParams && toParams.host_search) { let hasMoreThanDefaultKeys = false; angular.forEach(toParams.host_search, function(value, key) { - if(key !== 'order_by' && key !== 'page_size') { + if(key !== 'order_by' && key !== 'page_size' && key !== 'page') { hasMoreThanDefaultKeys = true; } }); $scope.enableSmartInventoryButton = hasMoreThanDefaultKeys ? true : false; + $scope.smartInventoryButtonTooltip = hasMoreThanDefaultKeys ? InventoryHostsStrings.get('smartinventorybutton.ENABLED_INSTRUCTIONS') : InventoryHostsStrings.get('smartinventorybutton.DISABLED_INSTRUCTIONS'); } else { $scope.enableSmartInventoryButton = false; + $scope.smartInventoryButtonTooltip = InventoryHostsStrings.get('smartinventorybutton.DISABLED_INSTRUCTIONS'); } }); @@ -114,5 +117,5 @@ function HostsList($scope, HostsList, $rootScope, GetBasePath, export default ['$scope', 'HostsList', '$rootScope', 'GetBasePath', 'rbacUiControlService', 'Dataset', '$state', '$filter', 'Prompt', 'Wait', - 'HostsService', 'SetStatus', 'canAdd', HostsList + 'HostsService', 'SetStatus', 'canAdd', 'InventoryHostsStrings', HostsList ]; diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/groups/related/nested-hosts/group-nested-hosts-list.controller.js b/awx/ui/client/src/inventories-hosts/inventories/related/groups/related/nested-hosts/group-nested-hosts-list.controller.js index f86a550218..056b61cf96 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/groups/related/nested-hosts/group-nested-hosts-list.controller.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/groups/related/nested-hosts/group-nested-hosts-list.controller.js @@ -6,10 +6,10 @@ export default ['$scope', 'NestedHostsListDefinition', '$rootScope', 'GetBasePath', 'rbacUiControlService', 'Dataset', '$state', '$filter', 'Prompt', 'Wait', - 'HostsService', 'SetStatus', 'canAdd', 'GroupsService', 'ProcessErrors', 'groupData', 'inventoryData', + 'HostsService', 'SetStatus', 'canAdd', 'GroupsService', 'ProcessErrors', 'groupData', 'inventoryData', 'InventoryHostsStrings', function($scope, NestedHostsListDefinition, $rootScope, GetBasePath, rbacUiControlService, Dataset, $state, $filter, Prompt, Wait, - HostsService, SetStatus, canAdd, GroupsService, ProcessErrors, groupData, inventoryData) { + HostsService, SetStatus, canAdd, GroupsService, ProcessErrors, groupData, inventoryData, InventoryHostsStrings) { let list = NestedHostsListDefinition; @@ -19,6 +19,7 @@ export default ['$scope', 'NestedHostsListDefinition', '$rootScope', 'GetBasePat $scope.canAdd = canAdd; $scope.enableSmartInventoryButton = false; $scope.disassociateFrom = groupData; + $scope.smartInventoryButtonTooltip = InventoryHostsStrings.get('smartinventorybutton.DISABLED_INSTRUCTIONS'); // Search init $scope.list = list; @@ -50,14 +51,16 @@ export default ['$scope', 'NestedHostsListDefinition', '$rootScope', 'GetBasePat if(toParams && toParams.host_search) { let hasMoreThanDefaultKeys = false; angular.forEach(toParams.host_search, function(value, key) { - if(key !== 'order_by' && key !== 'page_size') { + if(key !== 'order_by' && key !== 'page_size' && key !== 'page') { hasMoreThanDefaultKeys = true; } }); $scope.enableSmartInventoryButton = hasMoreThanDefaultKeys ? true : false; + $scope.smartInventoryButtonTooltip = hasMoreThanDefaultKeys ? InventoryHostsStrings.get('smartinventorybutton.ENABLED_INSTRUCTIONS') : InventoryHostsStrings.get('smartinventorybutton.DISABLED_INSTRUCTIONS'); } else { $scope.enableSmartInventoryButton = false; + $scope.smartInventoryButtonTooltip = InventoryHostsStrings.get('smartinventorybutton.DISABLED_INSTRUCTIONS'); } }); diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/hosts/list/host-list.controller.js b/awx/ui/client/src/inventories-hosts/inventories/related/hosts/list/host-list.controller.js index c19110b627..11507bb8c8 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/hosts/list/host-list.controller.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/hosts/list/host-list.controller.js @@ -7,10 +7,10 @@ // import HostsService from './../hosts/host.service'; export default ['$scope', 'ListDefinition', '$rootScope', 'GetBasePath', 'rbacUiControlService', 'Dataset', '$state', '$filter', 'Prompt', 'Wait', - 'HostsService', 'SetStatus', 'canAdd', 'i18n', + 'HostsService', 'SetStatus', 'canAdd', 'i18n', 'InventoryHostsStrings', function($scope, ListDefinition, $rootScope, GetBasePath, rbacUiControlService, Dataset, $state, $filter, Prompt, Wait, - HostsService, SetStatus, canAdd, i18n) { + HostsService, SetStatus, canAdd, i18n, InventoryHostsStrings) { let list = ListDefinition; @@ -19,6 +19,7 @@ export default ['$scope', 'ListDefinition', '$rootScope', 'GetBasePath', function init(){ $scope.canAdd = canAdd; $scope.enableSmartInventoryButton = false; + $scope.smartInventoryButtonTooltip = InventoryHostsStrings.get('smartinventorybutton.DISABLED_INSTRUCTIONS'); // Search init $scope.list = list; @@ -45,14 +46,16 @@ export default ['$scope', 'ListDefinition', '$rootScope', 'GetBasePath', if(toParams && toParams.host_search) { let hasMoreThanDefaultKeys = false; angular.forEach(toParams.host_search, function(value, key) { - if(key !== 'order_by' && key !== 'page_size') { + if(key !== 'order_by' && key !== 'page_size' && key !== 'page') { hasMoreThanDefaultKeys = true; } }); $scope.enableSmartInventoryButton = hasMoreThanDefaultKeys ? true : false; + $scope.smartInventoryButtonTooltip = hasMoreThanDefaultKeys ? InventoryHostsStrings.get('smartinventorybutton.ENABLED_INSTRUCTIONS') : InventoryHostsStrings.get('smartinventorybutton.DISABLED_INSTRUCTIONS'); } else { $scope.enableSmartInventoryButton = false; + $scope.smartInventoryButtonTooltip = InventoryHostsStrings.get('smartinventorybutton.DISABLED_INSTRUCTIONS'); } }); diff --git a/awx/ui/client/src/inventories-hosts/inventory-hosts.strings.js b/awx/ui/client/src/inventories-hosts/inventory-hosts.strings.js index c64b73a933..6c46f7c9d5 100644 --- a/awx/ui/client/src/inventories-hosts/inventory-hosts.strings.js +++ b/awx/ui/client/src/inventories-hosts/inventory-hosts.strings.js @@ -28,6 +28,11 @@ function InventoryHostsStrings (BaseString) { MISSING_PERMISSIONS: t.s('You do not have sufficient permissions to edit the host filter.') } }; + + ns.smartinventorybutton = { + DISABLED_INSTRUCTIONS: "Please enter at least one search term to create a new Smart Inventory.", + ENABLED_INSTRUCTIONS: "Create a new Smart Inventory from search results." + }; } InventoryHostsStrings.$inject = ['BaseStringService']; From 2cab6982c16213378d1e4899c82b0afa8ce96cea Mon Sep 17 00:00:00 2001 From: mabashian Date: Thu, 12 Oct 2017 17:01:32 -0400 Subject: [PATCH 045/141] Moved wait stop calls on jt form so that they fire right before reloading state --- .../add-job-template/job-template-add.controller.js | 1 - .../edit-job-template/job-template-edit.controller.js | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js b/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js index 9cd73e1da7..f88b1057df 100644 --- a/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js +++ b/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js @@ -341,7 +341,6 @@ Rest.post(data) .then(({data}) => { - Wait('stop'); if (data.related && data.related.callback) { Alert('Callback URL', `Host callbacks are enabled for this template. The callback URL is: diff --git a/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js b/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js index fa5d74219a..75bc0ecf2a 100644 --- a/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js +++ b/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js @@ -496,7 +496,7 @@ export default $scope.removeTemplateSaveSuccess(); } $scope.removeTemplateSaveSuccess = $scope.$on('templateSaveSuccess', function(e, data) { - Wait('stop'); + if (data.related && data.related.callback) { Alert('Callback URL', @@ -606,6 +606,7 @@ export default } $q.all(defers) .then(function() { + Wait('stop'); saveCompleted(); }); }); From fcd03fb1c21d77a775a1a2a3ab3af94367348b0a Mon Sep 17 00:00:00 2001 From: Marliana Lara Date: Fri, 13 Oct 2017 14:50:13 -0400 Subject: [PATCH 046/141] Fix job standard out error message word-wrap --- .../job-results-stdout/job-results-stdout.block.less | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui/client/src/job-results/job-results-stdout/job-results-stdout.block.less b/awx/ui/client/src/job-results/job-results-stdout/job-results-stdout.block.less index 3e9c505a76..d186677cd1 100644 --- a/awx/ui/client/src/job-results/job-results-stdout/job-results-stdout.block.less +++ b/awx/ui/client/src/job-results/job-results-stdout/job-results-stdout.block.less @@ -165,7 +165,7 @@ color: @default-interface-txt; display: inline-block; white-space: pre-wrap; - word-break: break-word; + word-break: break-all; width:100%; background-color: @default-secondary-bg; } From e8dbfa42cf57e2e22f649e59f57750cd47bb087c Mon Sep 17 00:00:00 2001 From: mabashian Date: Tue, 17 Oct 2017 10:10:34 -0400 Subject: [PATCH 047/141] Fixed disassociate host from group help text --- .../nested-groups/host-nested-groups-disassociate.partial.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/hosts/related/nested-groups/host-nested-groups-disassociate.partial.html b/awx/ui/client/src/inventories-hosts/inventories/related/hosts/related/nested-groups/host-nested-groups-disassociate.partial.html index 05566db742..b446955f8a 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/hosts/related/nested-groups/host-nested-groups-disassociate.partial.html +++ b/awx/ui/client/src/inventories-hosts/inventories/related/hosts/related/nested-groups/host-nested-groups-disassociate.partial.html @@ -5,7 +5,7 @@