From 35110ef738844207afacb92cd0ae493244b5f95a Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Wed, 30 Aug 2017 17:02:02 -0400 Subject: [PATCH 01/52] Prevent mistakenly truncate ANSI SGR code in job event stdout --- awx/api/serializers.py | 18 +++++++++++++++++- awx/main/constants.py | 3 +++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 15e66e0239..98a8896b73 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -38,7 +38,7 @@ from rest_framework.utils.serializer_helpers import ReturnList from polymorphic.models import PolymorphicModel # AWX -from awx.main.constants import SCHEDULEABLE_PROVIDERS +from awx.main.constants import SCHEDULEABLE_PROVIDERS, ANSI_SGR_PATTERN from awx.main.models import * # noqa from awx.main.access import get_user_capabilities from awx.main.fields import ImplicitRoleField @@ -3064,6 +3064,14 @@ class JobEventSerializer(BaseSerializer): max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY if max_bytes > 0 and 'stdout' in ret and len(ret['stdout']) >= max_bytes: ret['stdout'] = ret['stdout'][:(max_bytes - 1)] + u'\u2026' + set_count = 0 + reset_count = 0 + for m in ANSI_SGR_PATTERN.finditer(ret['stdout']): + if m.string[m.start():m.end()] == u'\u001b[0m': + reset_count += 1 + else: + set_count += 1 + ret['stdout'] += u'\u001b[0m' * (set_count - reset_count) return ret @@ -3095,6 +3103,14 @@ class AdHocCommandEventSerializer(BaseSerializer): max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY if max_bytes > 0 and 'stdout' in ret and len(ret['stdout']) >= max_bytes: ret['stdout'] = ret['stdout'][:(max_bytes - 1)] + u'\u2026' + set_count = 0 + reset_count = 0 + for m in ANSI_SGR_PATTERN.finditer(ret['stdout']): + if m.string[m.start():m.end()] == u'\u001b[0m': + reset_count += 1 + else: + set_count += 1 + ret['stdout'] += u'\u001b[0m' * (set_count - reset_count) return ret diff --git a/awx/main/constants.py b/awx/main/constants.py index bd00147415..10be060094 100644 --- a/awx/main/constants.py +++ b/awx/main/constants.py @@ -1,8 +1,11 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved. +import re + from django.utils.translation import ugettext_lazy as _ CLOUD_PROVIDERS = ('azure', 'azure_rm', 'ec2', 'gce', 'rax', 'vmware', 'openstack', 'satellite6', 'cloudforms') SCHEDULEABLE_PROVIDERS = CLOUD_PROVIDERS + ('custom', 'scm',) PRIVILEGE_ESCALATION_METHODS = [ ('sudo', _('Sudo')), ('su', _('Su')), ('pbrun', _('Pbrun')), ('pfexec', _('Pfexec')), ('dzdo', _('DZDO')), ('pmrun', _('Pmrun')), ('runas', _('Runas'))] +ANSI_SGR_PATTERN = re.compile(r'\x1b\[[0-9;]*m') From 68391301b0f7a25e83b11bf838678654b4d17b49 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Tue, 29 Aug 2017 10:37:08 -0400 Subject: [PATCH 02/52] Pick up workflow cornercase in get_user_capabilities --- awx/main/access.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/awx/main/access.py b/awx/main/access.py index badd3f415f..e7152ac608 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -22,7 +22,7 @@ from awx.main.models import * # noqa from awx.main.models.unified_jobs import ACTIVE_STATES from awx.main.models.mixins import ResourceMixin -from awx.conf.license import LicenseForbids +from awx.conf.license import LicenseForbids, feature_enabled __all__ = ['get_user_queryset', 'check_user_access', 'check_user_access_with_errors', 'user_accessible_objects', 'consumer_access', @@ -304,6 +304,10 @@ class BaseAccess(object): if validation_errors: user_capabilities[display_method] = False continue + elif isinstance(obj, (WorkflowJobTemplate, WorkflowJob)): + if not feature_enabled('workflows'): + user_capabilities[display_method] = (display_method == 'delete') + continue elif display_method == 'copy' and isinstance(obj, WorkflowJobTemplate) and obj.organization_id is None: user_capabilities[display_method] = self.user.is_superuser continue From 1cdab96e026332b43e512652c163ca4091961e4e Mon Sep 17 00:00:00 2001 From: Jared Tabor Date: Thu, 31 Aug 2017 15:37:49 -0700 Subject: [PATCH 03/52] showing job explanation if it wasn't "Previous Task Failed..." --- awx/ui/client/src/job-results/job-results.partial.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/job-results/job-results.partial.html b/awx/ui/client/src/job-results/job-results.partial.html index 7f5d653bca..e8ec250e31 100644 --- a/awx/ui/client/src/job-results/job-results.partial.html +++ b/awx/ui/client/src/job-results/job-results.partial.html @@ -81,9 +81,12 @@ +
+ {{ job.job_explanation }} +
{{task_detail | limitTo:explanationLimit}} - + ... Show More From 768c7ba3dc07375d7006a5375941b8c08c77df6f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 1 Sep 2017 15:14:15 -0400 Subject: [PATCH 04/52] bump azurerm dependencies to support Ansible 2.4 see: https://github.com/ansible/ansible-tower/issues/7470 --- awx/plugins/inventory/azure_rm.py | 107 +++++++++++++++++++------- requirements/requirements_ansible.in | 15 +++- requirements/requirements_ansible.txt | 52 +++++++------ 3 files changed, 122 insertions(+), 52 deletions(-) diff --git a/awx/plugins/inventory/azure_rm.py b/awx/plugins/inventory/azure_rm.py index 73b8b959d3..b3b7e1e904 100755 --- a/awx/plugins/inventory/azure_rm.py +++ b/awx/plugins/inventory/azure_rm.py @@ -49,6 +49,7 @@ Command line arguments: - tenant - ad_user - password + - cloud_environment Environment variables: - AZURE_PROFILE @@ -58,6 +59,7 @@ Environment variables: - AZURE_TENANT - AZURE_AD_USER - AZURE_PASSWORD + - AZURE_CLOUD_ENVIRONMENT Run for Specific Host ----------------------- @@ -190,22 +192,27 @@ import json import os import re import sys +import inspect +import traceback + from packaging.version import Version from os.path import expanduser +import ansible.module_utils.six.moves.urllib.parse as urlparse HAS_AZURE = True HAS_AZURE_EXC = None try: from msrestazure.azure_exceptions import CloudError + from msrestazure import azure_cloud from azure.mgmt.compute import __version__ as azure_compute_version from azure.common import AzureMissingResourceHttpError, AzureHttpError from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials - from azure.mgmt.network.network_management_client import NetworkManagementClient - from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient - from azure.mgmt.compute.compute_management_client import ComputeManagementClient + from azure.mgmt.network import NetworkManagementClient + from azure.mgmt.resource.resources import ResourceManagementClient + from azure.mgmt.compute import ComputeManagementClient except ImportError as exc: HAS_AZURE_EXC = exc HAS_AZURE = False @@ -218,7 +225,8 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict( secret='AZURE_SECRET', tenant='AZURE_TENANT', ad_user='AZURE_AD_USER', - password='AZURE_PASSWORD' + password='AZURE_PASSWORD', + cloud_environment='AZURE_CLOUD_ENVIRONMENT', ) AZURE_CONFIG_SETTINGS = dict( @@ -232,7 +240,7 @@ AZURE_CONFIG_SETTINGS = dict( group_by_tag='AZURE_GROUP_BY_TAG' ) -AZURE_MIN_VERSION = "0.30.0rc5" +AZURE_MIN_VERSION = "2.0.0" def azure_id_to_dict(id): @@ -249,6 +257,7 @@ class AzureRM(object): def __init__(self, args): self._args = args + self._cloud_environment = None self._compute_client = None self._resource_client = None self._network_client = None @@ -262,6 +271,26 @@ class AzureRM(object): self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " "or define a profile in ~/.azure/credentials.") + # if cloud_environment specified, look up/build Cloud object + raw_cloud_env = self.credentials.get('cloud_environment') + if not raw_cloud_env: + self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default + else: + # try to look up "well-known" values via the name attribute on azure_cloud members + all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)] + matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env] + if len(matched_clouds) == 1: + self._cloud_environment = matched_clouds[0] + elif len(matched_clouds) > 1: + self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env)) + else: + if not urlparse.urlparse(raw_cloud_env).scheme: + self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds])) + try: + self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env) + except Exception as e: + self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message)) + if self.credentials.get('subscription_id', None) is None: self.fail("Credentials did not include a subscription_id value.") self.log("setting subscription_id") @@ -272,16 +301,23 @@ class AzureRM(object): self.credentials.get('tenant') is not None: self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'], secret=self.credentials['secret'], - tenant=self.credentials['tenant']) + tenant=self.credentials['tenant'], + cloud_environment=self._cloud_environment) elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None: - self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password']) + tenant = self.credentials.get('tenant') + if not tenant: + tenant = 'common' + self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], + self.credentials['password'], + tenant=tenant, + cloud_environment=self._cloud_environment) else: self.fail("Failed to authenticate with provided credentials. Some attributes were missing. " "Credentials must include client_id, secret and tenant or ad_user and password.") def log(self, msg): if self.debug: - print (msg + u'\n') + print(msg + u'\n') def fail(self, msg): raise Exception(msg) @@ -341,6 +377,10 @@ class AzureRM(object): self.log('Received credentials from parameters.') return arg_credentials + if arg_credentials['ad_user'] is not None: + self.log('Received credentials from parameters.') + return arg_credentials + # try environment env_credentials = self._get_env_credentials() if env_credentials: @@ -372,7 +412,12 @@ class AzureRM(object): def network_client(self): self.log('Getting network client') if not self._network_client: - self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id) + self._network_client = NetworkManagementClient( + self.azure_credentials, + self.subscription_id, + base_url=self._cloud_environment.endpoints.resource_manager, + api_version='2017-06-01' + ) self._register('Microsoft.Network') return self._network_client @@ -380,14 +425,24 @@ class AzureRM(object): def rm_client(self): self.log('Getting resource manager client') if not self._resource_client: - self._resource_client = ResourceManagementClient(self.azure_credentials, self.subscription_id) + self._resource_client = ResourceManagementClient( + self.azure_credentials, + self.subscription_id, + base_url=self._cloud_environment.endpoints.resource_manager, + api_version='2017-05-10' + ) return self._resource_client @property def compute_client(self): self.log('Getting compute client') if not self._compute_client: - self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id) + self._compute_client = ComputeManagementClient( + self.azure_credentials, + self.subscription_id, + base_url=self._cloud_environment.endpoints.resource_manager, + api_version='2017-03-30' + ) self._register('Microsoft.Compute') return self._compute_client @@ -440,7 +495,7 @@ class AzureInventory(object): self.include_powerstate = False self.get_inventory() - print (self._json_format_dict(pretty=self._args.pretty)) + print(self._json_format_dict(pretty=self._args.pretty)) sys.exit(0) def _parse_cli_args(self): @@ -448,13 +503,13 @@ class AzureInventory(object): parser = argparse.ArgumentParser( description='Produce an Ansible Inventory file for an Azure subscription') parser.add_argument('--list', action='store_true', default=True, - help='List instances (default: True)') + help='List instances (default: True)') parser.add_argument('--debug', action='store_true', default=False, - help='Send debug messages to STDOUT') + help='Send debug messages to STDOUT') parser.add_argument('--host', action='store', - help='Get all information about an instance') + help='Get all information about an instance') parser.add_argument('--pretty', action='store_true', default=False, - help='Pretty print JSON output(default: False)') + help='Pretty print JSON output(default: False)') parser.add_argument('--profile', action='store', help='Azure profile contained in ~/.azure/credentials') parser.add_argument('--subscription_id', action='store', @@ -465,10 +520,12 @@ class AzureInventory(object): help='Azure Client Secret') parser.add_argument('--tenant', action='store', help='Azure Tenant Id') - parser.add_argument('--ad-user', action='store', + parser.add_argument('--ad_user', action='store', help='Active Directory User') parser.add_argument('--password', action='store', help='password') + parser.add_argument('--cloud_environment', action='store', + help='Azure Cloud Environment name or metadata discovery URL') parser.add_argument('--resource-groups', action='store', help='Return inventory for comma separated list of resource group names') parser.add_argument('--tags', action='store', @@ -486,8 +543,7 @@ class AzureInventory(object): try: virtual_machines = self._compute_client.virtual_machines.list(resource_group) except Exception as exc: - sys.exit("Error: fetching virtual machines for resource group {0} - {1}".format(resource_group, - str(exc))) + sys.exit("Error: fetching virtual machines for resource group {0} - {1}".format(resource_group, str(exc))) if self._args.host or self.tags: selected_machines = self._selected_machines(virtual_machines) self._load_machines(selected_machines) @@ -510,7 +566,7 @@ class AzureInventory(object): for machine in machines: id_dict = azure_id_to_dict(machine.id) - #TODO - The API is returning an ID value containing resource group name in ALL CAPS. If/when it gets + # TODO - The API is returning an ID value containing resource group name in ALL CAPS. If/when it gets # fixed, we should remove the .lower(). Opened Issue # #574: https://github.com/Azure/azure-sdk-for-python/issues/574 resource_group = id_dict['resourceGroups'].lower() @@ -538,7 +594,7 @@ class AzureInventory(object): mac_address=None, plan=(machine.plan.name if machine.plan else None), virtual_machine_size=machine.hardware_profile.vm_size, - computer_name=machine.os_profile.computer_name, + computer_name=(machine.os_profile.computer_name if machine.os_profile else None), provisioning_state=machine.provisioning_state, ) @@ -559,7 +615,7 @@ class AzureInventory(object): ) # Add windows details - if machine.os_profile.windows_configuration is not None: + if machine.os_profile is not None and machine.os_profile.windows_configuration is not None: host_vars['windows_auto_updates_enabled'] = \ machine.os_profile.windows_configuration.enable_automatic_updates host_vars['windows_timezone'] = machine.os_profile.windows_configuration.time_zone @@ -790,13 +846,10 @@ class AzureInventory(object): def main(): if not HAS_AZURE: - sys.exit("The Azure python sdk is not installed (try `pip install 'azure>=2.0.0rc5' --upgrade`) - {0}".format(HAS_AZURE_EXC)) - - if Version(azure_compute_version) < Version(AZURE_MIN_VERSION): - sys.exit("Expecting azure.mgmt.compute.__version__ to be {0}. Found version {1} " - "Do you have Azure >= 2.0.0rc5 installed? (try `pip install 'azure>=2.0.0rc5' --upgrade`)".format(AZURE_MIN_VERSION, azure_compute_version)) + sys.exit("The Azure python sdk is not installed (try `pip install 'azure>={0}' --upgrade`) - {1}".format(AZURE_MIN_VERSION, HAS_AZURE_EXC)) AzureInventory() + if __name__ == '__main__': main() diff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in index 3e17a968e4..c686e9545f 100644 --- a/requirements/requirements_ansible.in +++ b/requirements/requirements_ansible.in @@ -1,5 +1,18 @@ apache-libcloud==2.0.0 -azure==2.0.0rc6 +# azure deps from https://github.com/ansible/ansible/blob/fe1153c0afa1ffd648147af97454e900560b3532/packaging/requirements/requirements-azure.txt +azure-mgmt-compute>=2.0.0,<3 +azure-mgmt-network>=1.3.0,<2 +azure-mgmt-storage>=1.2.0,<2 +azure-mgmt-resource>=1.1.0,<2 +azure-storage>=0.35.1,<0.36 +azure-cli-core>=2.0.12,<3 +msrestazure>=0.4.11,<0.5 +azure-mgmt-dns>=1.0.1,<2 +azure-mgmt-keyvault>=0.40.0,<0.41 +azure-mgmt-batch>=4.1.0,<5 +azure-mgmt-sql>=0.7.1,<0.8 +azure-mgmt-web>=0.32.0,<0.33 +azure-mgmt-containerservice>=1.0.0 backports.ssl-match-hostname==3.5.0.1 kombu==3.0.37 boto==2.46.1 diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index f31c02928c..20c57308e8 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -4,30 +4,30 @@ # # pip-compile --output-file requirements/requirements_ansible.txt requirements/requirements_ansible.in # -adal==0.4.5 # via msrestazure +adal==0.4.7 # via azure-cli-core, msrestazure amqp==1.4.9 # via kombu anyjson==0.3.3 # via kombu apache-libcloud==2.0.0 appdirs==1.4.3 # via os-client-config, python-ironicclient, setuptools +applicationinsights==0.11.0 # via azure-cli-core +argcomplete==1.9.2 # via azure-cli-core asn1crypto==0.22.0 # via cryptography -azure-batch==1.0.0 # via azure -azure-common[autorest]==1.1.4 # via azure-batch, azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-keyvault, azure-mgmt-logic, azure-mgmt-network, azure-mgmt-redis, azure-mgmt-resource, azure-mgmt-scheduler, azure-mgmt-storage, azure-servicebus, azure-servicemanagement-legacy, azure-storage -azure-mgmt-batch==1.0.0 # via azure-mgmt -azure-mgmt-compute==0.30.0rc6 # via azure-mgmt -azure-mgmt-keyvault==0.30.0rc6 # via azure-mgmt -azure-mgmt-logic==1.0.0 # via azure-mgmt -azure-mgmt-network==0.30.0rc6 # via azure-mgmt -azure-mgmt-nspkg==2.0.0 # via azure-batch, azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-keyvault, azure-mgmt-logic, azure-mgmt-network, azure-mgmt-redis, azure-mgmt-resource, azure-mgmt-scheduler, azure-mgmt-storage -azure-mgmt-redis==1.0.0 # via azure-mgmt -azure-mgmt-resource==0.30.0rc6 # via azure-mgmt -azure-mgmt-scheduler==1.0.0 # via azure-mgmt -azure-mgmt-storage==0.30.0rc6 # via azure-mgmt -azure-mgmt==0.30.0rc6 # via azure -azure-nspkg==2.0.0 # via azure-common, azure-mgmt-nspkg, azure-storage -azure-servicebus==0.20.3 # via azure -azure-servicemanagement-legacy==0.20.4 # via azure -azure-storage==0.33.0 # via azure -azure==2.0.0rc6 +azure-cli-core==2.0.15 +azure-cli-nspkg==3.0.1 # via azure-cli-core +azure-common==1.1.8 # via azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-containerservice, azure-mgmt-dns, azure-mgmt-keyvault, azu +azure-mgmt-batch==4.1.0 +azure-mgmt-compute==2.1.0 +azure-mgmt-containerservice==1.0.0 +azure-mgmt-dns==1.0.1 +azure-mgmt-keyvault==0.40.0 +azure-mgmt-network==1.4.0 +azure-mgmt-nspkg==2.0.0 # via azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-containerservice, azure-mgmt-dns, azure-mgmt-keyvault, azu +azure-mgmt-resource==1.1.0 +azure-mgmt-sql==0.7.1 +azure-mgmt-storage==1.2.1 +azure-mgmt-web==0.32.0 +azure-nspkg==2.0.0 # via azure-cli-nspkg, azure-common, azure-mgmt-nspkg, azure-storage +azure-storage==0.35.1 babel==2.3.4 # via osc-lib, oslo.i18n, python-cinderclient, python-glanceclient, python-neutronclient, python-novaclient, python-openstackclient backports.ssl-match-hostname==3.5.0.1 boto3==1.4.4 @@ -37,7 +37,8 @@ certifi==2017.4.17 # via msrest cffi==1.10.0 # via cryptography cliff==2.7.0 # via osc-lib, python-designateclient, python-neutronclient, python-openstackclient cmd2==0.7.2 # via cliff -cryptography==1.9 # via adal, azure-storage, pyopenssl, secretstorage +colorama==0.3.9 # via azure-cli-core +cryptography==2.0.3 # via adal, azure-storage, paramiko, pyopenssl, secretstorage debtcollector==1.15.0 # via oslo.config, oslo.utils, python-designateclient, python-keystoneclient, python-neutronclient decorator==4.0.11 # via shade deprecation==1.0.1 # via openstacksdk @@ -47,6 +48,7 @@ enum34==1.1.6 # via cryptography, msrest funcsigs==1.0.2 # via debtcollector, oslo.utils functools32==3.2.3.post2 # via jsonschema futures==3.1.1 # via azure-storage, s3transfer, shade +humanfriendly==4.4.1 # via azure-cli-core idna==2.5 # via cryptography ipaddress==1.0.18 # via cryptography, shade iso8601==0.1.11 # via keystoneauth1, oslo.utils, python-neutronclient, python-novaclient @@ -61,8 +63,8 @@ kombu==3.0.37 lxml==3.8.0 # via pyvmomi monotonic==1.3 # via oslo.utils msgpack-python==0.4.8 # via oslo.serialization -msrest==0.4.10 # via azure-common, msrestazure -msrestazure==0.4.9 # via azure-common +msrest==0.4.14 # via azure-cli-core, msrestazure +msrestazure==0.4.13 munch==2.1.1 # via shade netaddr==0.7.19 # via oslo.config, oslo.utils, python-neutronclient netifaces==0.10.6 # via oslo.utils, shade @@ -83,9 +85,10 @@ prettytable==0.7.2 # via cliff, python-cinderclient, python-glanceclient, psphere==0.5.2 psutil==5.2.2 pycparser==2.17 # via cffi -pyjwt==1.5.0 # via adal +pygments==2.2.0 # via azure-cli-core +pyjwt==1.5.2 # via adal, azure-cli-core pykerberos==1.1.14 # via requests-kerberos -pyopenssl==17.0.0 # via pyvmomi +pyopenssl==17.2.0 # via azure-cli-core, python-glanceclient, pyvmomi pyparsing==2.2.0 # via cliff, cmd2, oslo.utils, packaging python-cinderclient==2.2.0 # via python-openstackclient, shade python-dateutil==2.6.0 # via adal, azure-storage, botocore @@ -114,6 +117,7 @@ simplejson==3.11.1 # via osc-lib, python-cinderclient, python-neutronclie six==1.10.0 # via cliff, cmd2, cryptography, debtcollector, keystoneauth1, munch, ntlm-auth, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, packaging, pyopenssl, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, pyvmomi, pywinrm, setuptools, shade, stevedore, warlock stevedore==1.23.0 # via cliff, keystoneauth1, openstacksdk, osc-lib, oslo.config, python-designateclient, python-keystoneclient suds==0.4 # via psphere +tabulate==0.7.7 # via azure-cli-core unicodecsv==0.14.1 # via cliff warlock==1.2.0 # via python-glanceclient wrapt==1.10.10 # via debtcollector, positional, python-glanceclient From 48e583026fb612ae125d7e02e1a6614043eb3305 Mon Sep 17 00:00:00 2001 From: Marliana Lara Date: Tue, 5 Sep 2017 18:01:41 -0400 Subject: [PATCH 05/52] Add margin between pagination pages and totals --- awx/ui/client/src/shared/paginate/paginate.block.less | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui/client/src/shared/paginate/paginate.block.less b/awx/ui/client/src/shared/paginate/paginate.block.less index 7adcf7c813..4cddec9d91 100644 --- a/awx/ui/client/src/shared/paginate/paginate.block.less +++ b/awx/ui/client/src/shared/paginate/paginate.block.less @@ -51,7 +51,7 @@ .Paginate-total { display: flex; align-items: flex-end; - margin-bottom: -2px; + margin: 0 0 -2px 10px; } .Paginate-filterLabel{ From 94d4a7e3f0c71717744ea7f75aaedcebfbe3f465 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Wed, 6 Sep 2017 10:39:49 -0400 Subject: [PATCH 06/52] Fix 7587: Remove credential_type_id in API v1 --- awx/api/serializers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 106e4b0198..c6553508fa 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -343,6 +343,8 @@ class BaseSerializer(serializers.ModelSerializer): continue summary_fields[fk] = OrderedDict() for field in related_fields: + if field == 'credential_type_id' and fk == 'credential' and self.version < 2: + continue fval = getattr(fkval, field, None) From 2f3124edf4756ca67ca0cc268230a53fb533d3f1 Mon Sep 17 00:00:00 2001 From: Marliana Lara Date: Thu, 7 Sep 2017 11:19:52 -0400 Subject: [PATCH 07/52] Remove modal backdrop on hidden modal event Signed-off-by: Marliana Lara --- awx/ui/client/src/shared/Utilities.js | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/ui/client/src/shared/Utilities.js b/awx/ui/client/src/shared/Utilities.js index 349898079f..08bdff71aa 100644 --- a/awx/ui/client/src/shared/Utilities.js +++ b/awx/ui/client/src/shared/Utilities.js @@ -120,6 +120,7 @@ angular.module('Utilities', ['RestServices', 'Utilities']) if (action) { action(); } + $('.modal-backdrop').remove(); }); $('#alert-modal').on('shown.bs.modal', function() { $('#alert_ok_btn').focus(); From 7965f9df6c39c0ed7efa4490c18742dc04a396b9 Mon Sep 17 00:00:00 2001 From: Marliana Lara Date: Thu, 7 Sep 2017 15:30:25 -0400 Subject: [PATCH 08/52] Fix form field lookup by populating the field name --- awx/ui/client/src/shared/lookup/lookup-modal.directive.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/awx/ui/client/src/shared/lookup/lookup-modal.directive.js b/awx/ui/client/src/shared/lookup/lookup-modal.directive.js index 978a09971b..a15d5b7d2b 100644 --- a/awx/ui/client/src/shared/lookup/lookup-modal.directive.js +++ b/awx/ui/client/src/shared/lookup/lookup-modal.directive.js @@ -29,10 +29,8 @@ export default ['templateUrl', function(templateUrl) { $scope.init = function() { let list = $scope.list; if($state.params.selected) { - $scope.currentSelection = { - name: null, - id: parseInt($state.params.selected) - }; + let selection = $scope[list.name].find(({id}) => id === parseInt($state.params.selected)); + $scope.currentSelection = _.pick(selection, 'id', 'name'); } $scope.$watch(list.name, function(){ selectRowIfPresent(); From 878e7ef49fcaf741c7e6f986ab51e574ca608269 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 31 Aug 2017 17:37:40 -0400 Subject: [PATCH 09/52] reap isolated jobs --- awx/main/scheduler/__init__.py | 72 +++++++++++++++++------- awx/main/tests/unit/test_task_manager.py | 2 +- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/awx/main/scheduler/__init__.py b/awx/main/scheduler/__init__.py index a6f477539c..5c713e6196 100644 --- a/awx/main/scheduler/__init__.py +++ b/awx/main/scheduler/__init__.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta import logging import uuid from sets import Set +import itertools # Django from django.conf import settings @@ -421,26 +422,40 @@ class TaskManager(): if not found_acceptable_queue: logger.debug("%s couldn't be scheduled on graph, waiting for next cycle", task.log_format) - def fail_jobs_if_not_in_celery(self, node_jobs, active_tasks, celery_task_start_time): + def fail_jobs_if_not_in_celery(self, node_jobs, active_tasks, celery_task_start_time, + isolated=False): for task in node_jobs: if (task.celery_task_id not in active_tasks and not hasattr(settings, 'IGNORE_CELERY_INSPECTOR')): if isinstance(task, WorkflowJob): continue if task.modified > celery_task_start_time: continue - task.status = 'failed' - task.job_explanation += ' '.join(( - 'Task was marked as running in Tower but was not present in', - 'Celery, so it has been marked as failed.', - )) + new_status = 'failed' + if isolated: + new_status = 'error' + task.status = new_status + if isolated: + # TODO: cancel and reap artifacts of lost jobs from heartbeat + task.job_explanation += ' '.join(( + 'Task was marked as running in Tower but its' + 'controller management daemon was not present in', + 'Celery, so it has been marked as failed.', + 'Task may still be running, but contactability is unknown.' + )) + else: + task.job_explanation += ' '.join(( + 'Task was marked as running in Tower but was not present in', + 'Celery, so it has been marked as failed.', + )) try: task.save(update_fields=['status', 'job_explanation']) except DatabaseError: logger.error("Task {} DB error in marking failed. Job possibly deleted.".format(task.log_format)) continue - awx_tasks._send_notification_templates(task, 'failed') - task.websocket_emit_status('failed') - logger.error("Task {} has no record in celery. Marking as failed".format(task.log_format)) + awx_tasks._send_notification_templates(task, new_status) + task.websocket_emit_status(new_status) + logger.error("{}Task {} has no record in celery. Marking as failed".format( + 'Isolated ' if isolated else '', task.log_format)) def cleanup_inconsistent_celery_tasks(self): ''' @@ -471,26 +486,41 @@ class TaskManager(): self.fail_jobs_if_not_in_celery(waiting_tasks, all_celery_task_ids, celery_task_start_time) for node, node_jobs in running_tasks.iteritems(): + isolated = False if node in active_queues: active_tasks = active_queues[node] else: ''' - Node task list not found in celery. If tower thinks the node is down - then fail all the jobs on the node. + Node task list not found in celery. We may branch into cases: + - instance is unknown to tower, system is improperly configured + - instance is reported as down, then fail all jobs on the node + - instance is an isolated node, then check running tasks + among all allowed controller nodes for management process ''' - try: - instance = Instance.objects.get(hostname=node) - if instance.capacity == 0: - active_tasks = [] + instance = Instance.objects.filter(hostname=node).first() + + if instance is None: + logger.error("Execution node Instance {} not found in database. " + "The node is currently executing jobs {}".format( + node, [j.log_format for j in node_jobs])) + active_tasks = [] + elif instance.capacity == 0: + active_tasks = [] + else: + cids = instance.rampart_groups.filter( + controller__isnull=False).values_list('id', flat=True) + if cids: + control_hosts = Instance.objects.filter( + rampart_groups__in=cids).values_list('hostname', flat=True) + active_tasks = set(itertools.chain(active_queues[host] for host in control_hosts)) + isolated = True else: continue - except Instance.DoesNotExist: - logger.error("Execution node Instance {} not found in database. " - "The node is currently executing jobs {}".format(node, - [j.log_format for j in node_jobs])) - active_tasks = [] - self.fail_jobs_if_not_in_celery(node_jobs, active_tasks, celery_task_start_time) + self.fail_jobs_if_not_in_celery( + node_jobs, active_tasks, celery_task_start_time, + isolated=isolated + ) def calculate_capacity_consumed(self, tasks): self.graph = InstanceGroup.objects.capacity_values(tasks=tasks, graph=self.graph) diff --git a/awx/main/tests/unit/test_task_manager.py b/awx/main/tests/unit/test_task_manager.py index 65b7607bb4..24536adae5 100644 --- a/awx/main/tests/unit/test_task_manager.py +++ b/awx/main/tests/unit/test_task_manager.py @@ -21,7 +21,7 @@ class TestCleanupInconsistentCeleryTasks(): @mock.patch.object(TaskManager, 'get_active_tasks', return_value=([], {})) @mock.patch.object(TaskManager, 'get_running_tasks', return_value=({'host1': [Job(id=2), Job(id=3),]}, [])) @mock.patch.object(InstanceGroup.objects, 'prefetch_related', return_value=[]) - @mock.patch.object(Instance.objects, 'get', side_effect=Instance.DoesNotExist) + @mock.patch.object(Instance.objects, 'prefetch_related', return_value=[]) @mock.patch('awx.main.scheduler.logger') def test_instance_does_not_exist(self, logger_mock, *args): logger_mock.error = mock.MagicMock(side_effect=RuntimeError("mocked")) From 8e1e60c18727d68d35440849cc1637696b4b7fa1 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 8 Sep 2017 10:01:44 -0700 Subject: [PATCH 10/52] simplify isolated job reaping by checking all task ids --- awx/main/scheduler/__init__.py | 18 ++++++------------ awx/main/tests/unit/test_task_manager.py | 2 +- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/awx/main/scheduler/__init__.py b/awx/main/scheduler/__init__.py index 5c713e6196..670012dc57 100644 --- a/awx/main/scheduler/__init__.py +++ b/awx/main/scheduler/__init__.py @@ -6,7 +6,6 @@ from datetime import datetime, timedelta import logging import uuid from sets import Set -import itertools # Django from django.conf import settings @@ -437,7 +436,7 @@ class TaskManager(): if isolated: # TODO: cancel and reap artifacts of lost jobs from heartbeat task.job_explanation += ' '.join(( - 'Task was marked as running in Tower but its' + 'Task was marked as running in Tower but its ', 'controller management daemon was not present in', 'Celery, so it has been marked as failed.', 'Task may still be running, but contactability is unknown.' @@ -452,7 +451,7 @@ class TaskManager(): except DatabaseError: logger.error("Task {} DB error in marking failed. Job possibly deleted.".format(task.log_format)) continue - awx_tasks._send_notification_templates(task, new_status) + awx_tasks._send_notification_templates(task, 'failed') task.websocket_emit_status(new_status) logger.error("{}Task {} has no record in celery. Marking as failed".format( 'Isolated ' if isolated else '', task.log_format)) @@ -506,16 +505,11 @@ class TaskManager(): active_tasks = [] elif instance.capacity == 0: active_tasks = [] + elif instance.rampart_groups.filter(controller__isnull=False).exists(): + active_tasks = all_celery_task_ids + isolated = True else: - cids = instance.rampart_groups.filter( - controller__isnull=False).values_list('id', flat=True) - if cids: - control_hosts = Instance.objects.filter( - rampart_groups__in=cids).values_list('hostname', flat=True) - active_tasks = set(itertools.chain(active_queues[host] for host in control_hosts)) - isolated = True - else: - continue + continue self.fail_jobs_if_not_in_celery( node_jobs, active_tasks, celery_task_start_time, diff --git a/awx/main/tests/unit/test_task_manager.py b/awx/main/tests/unit/test_task_manager.py index 24536adae5..1937b7b5ca 100644 --- a/awx/main/tests/unit/test_task_manager.py +++ b/awx/main/tests/unit/test_task_manager.py @@ -21,7 +21,7 @@ class TestCleanupInconsistentCeleryTasks(): @mock.patch.object(TaskManager, 'get_active_tasks', return_value=([], {})) @mock.patch.object(TaskManager, 'get_running_tasks', return_value=({'host1': [Job(id=2), Job(id=3),]}, [])) @mock.patch.object(InstanceGroup.objects, 'prefetch_related', return_value=[]) - @mock.patch.object(Instance.objects, 'prefetch_related', return_value=[]) + @mock.patch.object(Instance.objects, 'filter', return_value=mock.MagicMock(first=lambda: None)) @mock.patch('awx.main.scheduler.logger') def test_instance_does_not_exist(self, logger_mock, *args): logger_mock.error = mock.MagicMock(side_effect=RuntimeError("mocked")) From 42ee804464e0c859d28993c4433d78a23032af90 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 8 Sep 2017 14:36:17 -0700 Subject: [PATCH 11/52] add help for instance provisioning --- awx/main/management/commands/deprovision_instance.py | 5 +++++ awx/main/management/commands/provision_instance.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/awx/main/management/commands/deprovision_instance.py b/awx/main/management/commands/deprovision_instance.py index 117c615db5..bd26bb74f0 100644 --- a/awx/main/management/commands/deprovision_instance.py +++ b/awx/main/management/commands/deprovision_instance.py @@ -17,6 +17,11 @@ class Command(BaseCommand): Deprovision a Tower cluster node """ + help = ( + 'Remove instance from the database. ' + 'Specify `--hostname` to use this command.' + ) + option_list = BaseCommand.option_list + ( make_option('--hostname', dest='hostname', type='string', help='Hostname used during provisioning'), diff --git a/awx/main/management/commands/provision_instance.py b/awx/main/management/commands/provision_instance.py index 98831dfb52..24aad2aebb 100644 --- a/awx/main/management/commands/provision_instance.py +++ b/awx/main/management/commands/provision_instance.py @@ -16,6 +16,11 @@ class Command(BaseCommand): Regsiter this instance with the database for HA tracking. """ + help = ( + 'Add instance to the database. ' + 'Specify `--hostname` to use this command.' + ) + option_list = BaseCommand.option_list + ( make_option('--hostname', dest='hostname', type='string', help='Hostname used during provisioning'), From 11b06a2e5ef54dc469c9a3563cc46a0cab65a87a Mon Sep 17 00:00:00 2001 From: John Mitchell Date: Fri, 8 Sep 2017 17:55:41 -0400 Subject: [PATCH 12/52] add disassociation disclaimer --- .../hosts/related/groups/hosts-related-groups.partial.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/inventories-hosts/hosts/related/groups/hosts-related-groups.partial.html b/awx/ui/client/src/inventories-hosts/hosts/related/groups/hosts-related-groups.partial.html index 4660f008cb..437ff3a242 100644 --- a/awx/ui/client/src/inventories-hosts/hosts/related/groups/hosts-related-groups.partial.html +++ b/awx/ui/client/src/inventories-hosts/hosts/related/groups/hosts-related-groups.partial.html @@ -19,7 +19,10 @@
-
Are you sure you want to disassociate the host below from {{disassociateGroup.name}}?
+
+ Are you sure you want to disassociate the host below from {{disassociateGroup.name}}?

+ Note that only hosts directly in this group can be disassociated. Hosts in sub-groups must be disassociated directly from the sub-group level that they belong. +
{{ host.name }}
From 6f26f88bbd4f639210d3de031fd481cf793b6845 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 11 Sep 2017 10:48:32 -0400 Subject: [PATCH 13/52] Add a TODO for 3.3 cleanup --- awx/api/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index c6553508fa..0013975eb5 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -343,7 +343,7 @@ class BaseSerializer(serializers.ModelSerializer): continue summary_fields[fk] = OrderedDict() for field in related_fields: - if field == 'credential_type_id' and fk == 'credential' and self.version < 2: + if field == 'credential_type_id' and fk == 'credential' and self.version < 2: # TODO: remove version check in 3.3 continue fval = getattr(fkval, field, None) From 4cc58a221b888449f4e1ce89de2f3c7d8fe249b5 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 11 Sep 2017 11:28:43 -0400 Subject: [PATCH 14/52] fix busted conf unit tests --- awx/conf/tests/unit/test_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/conf/tests/unit/test_settings.py b/awx/conf/tests/unit/test_settings.py index 2f3c197478..d290228056 100644 --- a/awx/conf/tests/unit/test_settings.py +++ b/awx/conf/tests/unit/test_settings.py @@ -264,7 +264,7 @@ def test_setting_from_db_with_unicode(settings, mocker, encrypted): # this simulates a bug in python-memcached; see https://github.com/linsomniac/python-memcached/issues/79 value = six.u('Iñtërnâtiônàlizætiøn').encode('utf-8') - setting_from_db = mocker.Mock(key='AWX_SOME_SETTING', value=value) + setting_from_db = mocker.Mock(id=1, key='AWX_SOME_SETTING', value=value) mocks = mocker.Mock(**{ 'order_by.return_value': mocker.Mock(**{ '__iter__': lambda self: iter([setting_from_db]), From d38264660ef2696bd29c04314daa74f070a502f5 Mon Sep 17 00:00:00 2001 From: mabashian Date: Mon, 11 Sep 2017 11:40:35 -0400 Subject: [PATCH 15/52] Updated error message when can_update comes back false on an inventory source update GET Signed-off-by: mabashian --- .../job-submission-factories/inventory-update.factory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui/client/src/job-submission/job-submission-factories/inventory-update.factory.js b/awx/ui/client/src/job-submission/job-submission-factories/inventory-update.factory.js index cb4c0bf983..8b0ebe3fd2 100644 --- a/awx/ui/client/src/job-submission/job-submission-factories/inventory-update.factory.js +++ b/awx/ui/client/src/job-submission/job-submission-factories/inventory-update.factory.js @@ -67,7 +67,7 @@ export default } } else { Wait('stop'); - Alert('Permission Denied', 'You do not have access to run the inventory sync. Please contact your system administrator.', + Alert('Error Launching Sync', 'Unable to execute the inventory sync. Please contact your system administrator.', 'alert-danger'); } } From a2ca0e6012e860d329554c86ef852d51b7e64dec Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 24 Aug 2017 16:37:59 -0400 Subject: [PATCH 16/52] add process isolation to project updates see: https://github.com/ansible/ansible-tower/issues/7506 --- awx/main/tasks.py | 12 ++++++++++++ awx/main/tests/unit/test_tasks.py | 20 ++++++++++++++++++++ awx/main/utils/common.py | 1 + 3 files changed, 33 insertions(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 2c884802c2..63e597ef0c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -478,6 +478,7 @@ class BaseTask(LogErrorsTask): model = None abstract = True cleanup_paths = [] + proot_show_paths = [] def update_model(self, pk, _attempt=0, **updates): """Reload the model instance from the database and update the @@ -793,6 +794,7 @@ class BaseTask(LogErrorsTask): # May have to serialize the value kwargs['private_data_files'] = self.build_private_data_files(instance, **kwargs) kwargs['passwords'] = self.build_passwords(instance, **kwargs) + kwargs['proot_show_paths'] = self.proot_show_paths args = self.build_args(instance, **kwargs) safe_args = self.build_safe_args(instance, **kwargs) output_replacements = self.build_output_replacements(instance, **kwargs) @@ -1288,6 +1290,10 @@ class RunProjectUpdate(BaseTask): name = 'awx.main.tasks.run_project_update' model = ProjectUpdate + @property + def proot_show_paths(self): + return [settings.PROJECTS_ROOT] + def build_private_data(self, project_update, **kwargs): ''' Return SSH private key data needed for this project update. @@ -1594,6 +1600,12 @@ class RunProjectUpdate(BaseTask): if status == 'successful' and instance.launch_type != 'sync': self._update_dependent_inventories(instance, dependent_inventory_sources) + def should_use_proot(self, instance, **kwargs): + ''' + Return whether this task should use proot. + ''' + return getattr(settings, 'AWX_PROOT_ENABLED', False) + class RunInventoryUpdate(BaseTask): diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 99ad34d766..d262e4b1ac 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1096,6 +1096,26 @@ class TestProjectUpdateCredentials(TestJobExecution): ] } + def test_bwrap_exposes_projects_root(self): + ssh = CredentialType.defaults['ssh']() + self.instance.scm_type = 'git' + self.instance.credential = Credential( + pk=1, + credential_type=ssh, + ) + self.task.run(self.pk) + + assert self.run_pexpect.call_count == 1 + call_args, call_kwargs = self.run_pexpect.call_args_list[0] + args, cwd, env, stdout = call_args + + assert ' '.join(args).startswith('bwrap') + ' '.join([ + '--bind', + settings.PROJECTS_ROOT, + settings.PROJECTS_ROOT, + ]) in ' '.join(args) + def test_username_and_password_auth(self, scm_type): ssh = CredentialType.defaults['ssh']() self.instance.scm_type = scm_type diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 58d795567f..11869e846b 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -701,6 +701,7 @@ def wrap_args_with_proot(args, cwd, **kwargs): show_paths = [cwd] show_paths.extend([settings.ANSIBLE_VENV_PATH, settings.AWX_VENV_PATH]) show_paths.extend(getattr(settings, 'AWX_PROOT_SHOW_PATHS', None) or []) + show_paths.extend(kwargs.get('proot_show_paths', [])) for path in sorted(set(show_paths)): if not os.path.exists(path): continue From a9c9ecb5ea76de51d4fe8519e54b1e834963e6fd Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 6 Sep 2017 14:12:47 -0700 Subject: [PATCH 17/52] bind ansible and awx virtualenvs readonly so that jobs can't modify them see: https://github.com/ansible/ansible-tower/issues/7558 --- awx/main/tests/unit/test_tasks.py | 9 +++++++++ awx/main/utils/common.py | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index d262e4b1ac..6038440357 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -281,6 +281,15 @@ class TestGenericRun(TestJobExecution): args, cwd, env, stdout = call_args assert args[0] == 'bwrap' + def test_bwrap_virtualenvs_are_readonly(self): + self.task.run(self.pk) + + assert self.run_pexpect.call_count == 1 + call_args, _ = self.run_pexpect.call_args_list[0] + args, cwd, env, stdout = call_args + assert '--ro-bind %s %s' % (settings.ANSIBLE_VENV_PATH, settings.ANSIBLE_VENV_PATH) in ' '.join(args) # noqa + assert '--ro-bind %s %s' % (settings.AWX_VENV_PATH, settings.AWX_VENV_PATH) in ' '.join(args) # noqa + def test_awx_task_env(self): patch = mock.patch('awx.main.tasks.settings.AWX_TASK_ENV', {'FOO': 'BAR'}) patch.start() diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 11869e846b..c8c382472a 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -699,7 +699,11 @@ def wrap_args_with_proot(args, cwd, **kwargs): show_paths = [cwd, kwargs['private_data_dir']] else: show_paths = [cwd] - show_paths.extend([settings.ANSIBLE_VENV_PATH, settings.AWX_VENV_PATH]) + for venv in ( + settings.ANSIBLE_VENV_PATH, + settings.AWX_VENV_PATH + ): + new_args.extend(['--ro-bind', venv, venv]) show_paths.extend(getattr(settings, 'AWX_PROOT_SHOW_PATHS', None) or []) show_paths.extend(kwargs.get('proot_show_paths', [])) for path in sorted(set(show_paths)): From 4213960ec37e0707071f4fe5dab25379e65470b6 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 11 Sep 2017 17:18:43 -0400 Subject: [PATCH 18/52] write the scm_revision_output to the project path instead of /tmp see: https://github.com/ansible/ansible-tower/issues/7558 --- awx/main/tasks.py | 2 +- awx/main/tests/unit/test_tasks.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 63e597ef0c..1a6e0f25c1 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1307,7 +1307,7 @@ class RunProjectUpdate(BaseTask): } } ''' - handle, self.revision_path = tempfile.mkstemp(dir=settings.AWX_PROOT_BASE_PATH) + handle, self.revision_path = tempfile.mkstemp(dir=settings.PROJECTS_ROOT) self.cleanup_paths.append(self.revision_path) private_data = {'credentials': {}} if project_update.credential: diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 6038440357..bbc52fad8d 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -181,6 +181,8 @@ class TestJobExecution: EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY-----' def setup_method(self, method): + if not os.path.exists(settings.PROJECTS_ROOT): + os.mkdir(settings.PROJECTS_ROOT) self.project_path = tempfile.mkdtemp(prefix='awx_project_') with open(os.path.join(self.project_path, 'helloworld.yml'), 'w') as f: f.write('---') @@ -1124,6 +1126,7 @@ class TestProjectUpdateCredentials(TestJobExecution): settings.PROJECTS_ROOT, settings.PROJECTS_ROOT, ]) in ' '.join(args) + assert '"scm_revision_output": "/projects/tmp' in ' '.join(args) def test_username_and_password_auth(self, scm_type): ssh = CredentialType.defaults['ssh']() From 9f3a0c071697ef395e53fd4190d53f5d73f14864 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 12 Sep 2017 09:42:50 -0400 Subject: [PATCH 19/52] Fix an issue where dependent updates weren't sorted correctly When considering previous / current Project Updates we weren't properly sorting the previous runs. We also make sure we filter down to just "check" style project updates and don't consider 'run' style standalone project updates when deciding what are potentially related project updates --- awx/main/scheduler/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/main/scheduler/__init__.py b/awx/main/scheduler/__init__.py index 670012dc57..549f1546f3 100644 --- a/awx/main/scheduler/__init__.py +++ b/awx/main/scheduler/__init__.py @@ -300,7 +300,7 @@ class TaskManager(): # Already processed dependencies for this job if job.dependent_jobs.all(): return False - latest_inventory_update = InventoryUpdate.objects.filter(inventory_source=inventory_source).order_by("created") + latest_inventory_update = InventoryUpdate.objects.filter(inventory_source=inventory_source).order_by("-created") if not latest_inventory_update.exists(): return True latest_inventory_update = latest_inventory_update.first() @@ -323,7 +323,7 @@ class TaskManager(): now = tz_now() if job.dependent_jobs.all(): return False - latest_project_update = ProjectUpdate.objects.filter(project=job.project).order_by("created") + latest_project_update = ProjectUpdate.objects.filter(project=job.project, job_type='check').order_by("-created") if not latest_project_update.exists(): return True latest_project_update = latest_project_update.first() From 5009f283d53e07f7f629379b1cf7db5f6bbd5cb7 Mon Sep 17 00:00:00 2001 From: John Mitchell Date: Tue, 12 Sep 2017 12:01:06 -0400 Subject: [PATCH 20/52] always start sync all updates --- .../inventory-update.factory.js | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/awx/ui/client/src/job-submission/job-submission-factories/inventory-update.factory.js b/awx/ui/client/src/job-submission/job-submission-factories/inventory-update.factory.js index cb4c0bf983..4bd0046dd2 100644 --- a/awx/ui/client/src/job-submission/job-submission-factories/inventory-update.factory.js +++ b/awx/ui/client/src/job-submission/job-submission-factories/inventory-update.factory.js @@ -44,16 +44,7 @@ export default Rest.get() .success(function (data) { if(params.updateAllSources) { - let userCanUpdateAllSources = true; - _.forEach(data, function(inventory_source){ - if (!inventory_source.can_update) { - userCanUpdateAllSources = false; - } - }); - - if(userCanUpdateAllSources) { - scope.$emit('StartTheUpdate', {}); - } + scope.$emit('StartTheUpdate', {}); } else { inventory_source = data; From ae5b309de09c9f4f87df8d6ac07876fd8d00df5d Mon Sep 17 00:00:00 2001 From: Jared Tabor Date: Tue, 12 Sep 2017 10:21:52 -0700 Subject: [PATCH 21/52] catching org save error the org PUT was missing a .catch, however the edit controller already had it. --- .../organizations/add/organizations-add.controller.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/organizations/add/organizations-add.controller.js b/awx/ui/client/src/organizations/add/organizations-add.controller.js index 2a43f8652d..fbd0957153 100644 --- a/awx/ui/client/src/organizations/add/organizations-add.controller.js +++ b/awx/ui/client/src/organizations/add/organizations-add.controller.js @@ -54,9 +54,16 @@ export default ['$scope', '$rootScope', '$location', '$stateParams', .catch(({data, status}) => { ProcessErrors($scope, data, status, form, { hdr: 'Error!', - msg: 'Failed to add new organization. Post returned status: ' + status + msg: 'Failed to save instance groups. POST returned status: ' + status }); }); + }) + .catch(({data, status}) => { + let explanation = _.has(data, "name") ? data.name[0] : ""; + ProcessErrors($scope, data, status, OrganizationForm, { + hdr: 'Error!', + msg: `Failed to save organization. PUT status: ${status}. ${explanation}` + }); }); }; From 87eab79f7070034eef7a6367d5a6451fe68d8e8e Mon Sep 17 00:00:00 2001 From: gconsidine Date: Tue, 12 Sep 2017 14:57:07 -0400 Subject: [PATCH 22/52] Add support for quoted values containing spaces in search --- .../smart-search/smart-search.controller.js | 11 +++++-- .../smart-search/smart-search.service.js | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/awx/ui/client/src/shared/smart-search/smart-search.controller.js b/awx/ui/client/src/shared/smart-search/smart-search.controller.js index 5b47b77ece..c3898dac41 100644 --- a/awx/ui/client/src/shared/smart-search/smart-search.controller.js +++ b/awx/ui/client/src/shared/smart-search/smart-search.controller.js @@ -161,10 +161,15 @@ export default ['$stateParams', '$scope', '$state', 'GetBasePath', 'QuerySet', ' terms = (terms) ? terms.trim() : ""; if(terms && terms !== '') { - // Split the terms up - let splitTerms = SmartSearchService.splitSearchIntoTerms(terms); - _.forEach(splitTerms, (term) => { + let splitTerms; + if ($scope.singleSearchParam === 'host_filter') { + splitTerms = SmartSearchService.splitHostIntoTerms(terms); + } else { + splitTerms = SmartSearchService.splitSearchIntoTerms(terms); + } + + _.forEach(splitTerms, (term) => { let termParts = SmartSearchService.splitTermIntoParts(term); function combineSameSearches(a,b){ diff --git a/awx/ui/client/src/shared/smart-search/smart-search.service.js b/awx/ui/client/src/shared/smart-search/smart-search.service.js index 48681b1e0d..1fea6ab4c6 100644 --- a/awx/ui/client/src/shared/smart-search/smart-search.service.js +++ b/awx/ui/client/src/shared/smart-search/smart-search.service.js @@ -1,5 +1,34 @@ export default [function() { return { + /** + * For the Smart Host Filter, values with spaces are wrapped with double quotes on input. + * To avoid having these quoted values split up and treated as terms themselves, some + * work is done to encode quotes in quoted values and the spaces within those quoted + * values before calling to `splitSearchIntoTerms`. + */ + splitHostIntoTerms (searchString) { + let groups = []; + let quoted; + + searchString.split(' ').forEach(substring => { + if (substring.includes(':"')) { + quoted = substring; + } else if (quoted) { + quoted += ` ${substring}`; + + if (substring.includes('"')) { + quoted = quoted.replace(/"/g, encodeURIComponent('"')); + quoted = quoted.replace(/ /g, encodeURIComponent(' ')); + groups.push(quoted); + quoted = undefined; + } + } else { + groups.push(substring); + } + }); + + return this.splitSearchIntoTerms(groups.join(' ')); + }, splitSearchIntoTerms(searchString) { return searchString.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g); }, From d308946360d69603b57b1f9e111d87bcdff3fd32 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 12 Sep 2017 15:16:05 -0400 Subject: [PATCH 23/52] Allow system auditor to set their own password --- awx/main/access.py | 6 ++++-- awx/main/tests/functional/test_rbac_user.py | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index d73b877389..257a6b8a84 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -463,8 +463,10 @@ class UserAccess(BaseAccess): def can_change(self, obj, data): if data is not None and ('is_superuser' in data or 'is_system_auditor' in data): - if (to_python_boolean(data.get('is_superuser', 'false'), allow_none=True) or - to_python_boolean(data.get('is_system_auditor', 'false'), allow_none=True)) and not self.user.is_superuser: + if to_python_boolean(data.get('is_superuser', 'false'), allow_none=True) and \ + not self.user.is_superuser: + return False + if to_python_boolean(data.get('is_system_auditor', 'false'), allow_none=True) and not (self.user.is_superuser or self.user == obj): return False # A user can be changed if they are themselves, or by org admins or # superusers. Change permission implies changing only certain fields diff --git a/awx/main/tests/functional/test_rbac_user.py b/awx/main/tests/functional/test_rbac_user.py index 8f307ea0e3..bbfe0267cd 100644 --- a/awx/main/tests/functional/test_rbac_user.py +++ b/awx/main/tests/functional/test_rbac_user.py @@ -44,6 +44,12 @@ def test_system_auditor_is_system_auditor(system_auditor): assert system_auditor.is_system_auditor +@pytest.mark.django_db +def test_system_auditor_can_modify_self(system_auditor): + access = UserAccess(system_auditor) + assert access.can_change(obj=system_auditor, data=dict(is_system_auditor='true')) + + @pytest.mark.django_db def test_user_queryset(user): u = user('pete', False) From bd42dfe4749bc613afca2d0b13588a44f023bc33 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 12 Sep 2017 16:35:49 -0400 Subject: [PATCH 24/52] defer UnifiedJob.result_stdout_text for improved performance result_stdout_text can be _very_ large - some customers have 5MB+ per job; querying for this in list contexts results in _very_ large datasets being read from the database which is very slow. It's very uncommon to actually need this column outside of the context of job details, so defer it. see: https://github.com/ansible/ansible-tower/issues/7568 --- awx/main/access.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/awx/main/access.py b/awx/main/access.py index d73b877389..61c6a54cbe 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -105,7 +105,14 @@ def get_user_capabilities(user, instance, **kwargs): convenient for the user interface to consume and hide or show various actions in the interface. ''' - access_class = access_registry[instance.__class__] + cls = instance.__class__ + # When `.defer()` is used w/ the Django ORM, the result is a subclass of + # the original that represents e.g., + # awx.main.models.ad_hoc_commands.AdHocCommand_Deferred_result_stdout_text + # We want to do the access registry lookup keyed on the base class name. + if getattr(cls, '_deferred', False): + cls = instance.__class__.__bases__[0] + access_class = access_registry[cls] return access_class(user).get_user_capabilities(instance, **kwargs) @@ -2071,6 +2078,7 @@ class UnifiedJobAccess(BaseAccess): # 'job_template__project', # 'job_template__credential', #) + qs = qs.defer('result_stdout_text') return qs.all() From 28fa5077d5faa29c1785035f660190b3441dbefc Mon Sep 17 00:00:00 2001 From: mabashian Date: Wed, 13 Sep 2017 11:38:56 -0400 Subject: [PATCH 25/52] Fixed jt admin edit bug Signed-off-by: mabashian --- .../job-template-add.controller.js | 2 + .../job-template-edit.controller.js | 196 +++++++++++++++- .../factories/callback-help-init.factory.js | 214 +----------------- .../job_templates/job-template.form.js | 7 +- .../multi-credential-modal.directive.js | 26 ++- .../multi-credential-modal.partial.html | 6 +- .../multi-credential.block.less | 4 + .../multi-credential.partial.html | 4 +- .../multi-credential.service.js | 4 + awx/ui/client/src/templates/main.js | 103 +++++++-- 10 files changed, 314 insertions(+), 252 deletions(-) diff --git a/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js b/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js index 85d7480957..1438f89f37 100644 --- a/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js +++ b/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js @@ -37,6 +37,8 @@ $scope.mode = "add"; $scope.parseType = 'yaml'; $scope.credentialNotPresent = false; + $scope.canChangeProject = true; + $scope.canChangeInventory = true; md5Setup({ scope: $scope, diff --git a/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js b/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js index 0884a04ca2..5e38774766 100644 --- a/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js +++ b/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js @@ -18,13 +18,15 @@ export default 'Empty', 'Prompt', 'ToJSON', 'GetChoices', 'CallbackHelpInit', 'InitiatePlaybookRun' , 'initSurvey', '$state', 'CreateSelect2', 'ToggleNotification','$q', 'InstanceGroupsService', 'InstanceGroupsData', 'MultiCredentialService', 'availableLabels', + 'canChangeProject', 'canChangeInventory', 'jobTemplateData', 'ParseVariableString', function( $filter, $scope, $rootScope, $location, $stateParams, JobTemplateForm, GenerateForm, Rest, Alert, ProcessErrors, GetBasePath, md5Setup, ParseTypeChange, Wait, selectedLabels, i18n, Empty, Prompt, ToJSON, GetChoices, CallbackHelpInit, InitiatePlaybookRun, SurveyControllerInit, $state, - CreateSelect2, ToggleNotification, $q, InstanceGroupsService, InstanceGroupsData, MultiCredentialService, availableLabels + CreateSelect2, ToggleNotification, $q, InstanceGroupsService, InstanceGroupsData, MultiCredentialService, availableLabels, + canChangeProject, canChangeInventory, jobTemplateData, ParseVariableString ) { $scope.$watch('job_template_obj.summary_fields.user_capabilities.edit', function(val) { @@ -47,6 +49,7 @@ export default function init() { CallbackHelpInit({ scope: $scope }); + $scope.playbook_options = null; $scope.playbook = null; $scope.mode = 'edit'; @@ -57,6 +60,8 @@ export default $scope.surveyTooltip = i18n._('Surveys allow users to be prompted at job launch with a series of questions related to the job. This allows for variables to be defined that affect the playbook run at time of launch.'); $scope.job_tag_options = []; $scope.skip_tag_options = []; + $scope.canChangeProject = canChangeProject; + $scope.canChangeInventory = canChangeInventory; SurveyControllerInit({ scope: $scope, @@ -253,7 +258,194 @@ export default $scope.rmoveLoadJobs(); } $scope.removeLoadJobs = $scope.$on('LoadJobs', function() { - $scope.fillJobTemplate(); + $scope.job_template_obj = jobTemplateData; + $scope.name = jobTemplateData.name; + var fld, i; + for (fld in form.fields) { + if (fld !== 'variables' && fld !== 'survey' && fld !== 'forks' && jobTemplateData[fld] !== null && jobTemplateData[fld] !== undefined) { + if (form.fields[fld].type === 'select') { + if ($scope[fld + '_options'] && $scope[fld + '_options'].length > 0) { + for (i = 0; i < $scope[fld + '_options'].length; i++) { + if (jobTemplateData[fld] === $scope[fld + '_options'][i].value) { + $scope[fld] = $scope[fld + '_options'][i]; + } + } + } else { + $scope[fld] = jobTemplateData[fld]; + } + } else { + $scope[fld] = jobTemplateData[fld]; + if(!Empty(jobTemplateData.summary_fields.survey)) { + $scope.survey_exists = true; + } + } + master[fld] = $scope[fld]; + } + if (fld === 'forks') { + if (jobTemplateData[fld] !== 0) { + $scope[fld] = jobTemplateData[fld]; + master[fld] = $scope[fld]; + } + } + if (fld === 'variables') { + // Parse extra_vars, converting to YAML. + $scope.variables = ParseVariableString(jobTemplateData.extra_vars); + master.variables = $scope.variables; + } + if (form.fields[fld].type === 'lookup' && jobTemplateData.summary_fields[form.fields[fld].sourceModel]) { + $scope[form.fields[fld].sourceModel + '_' + form.fields[fld].sourceField] = + jobTemplateData.summary_fields[form.fields[fld].sourceModel][form.fields[fld].sourceField]; + master[form.fields[fld].sourceModel + '_' + form.fields[fld].sourceField] = + $scope[form.fields[fld].sourceModel + '_' + form.fields[fld].sourceField]; + } + if (form.fields[fld].type === 'checkbox_group') { + for(var j=0; j ({name: i, label: i, value: i})) : []; + $scope.job_tags = $scope.job_tag_options; + master.job_tags = $scope.job_tags; + + $scope.skip_tag_options = (jobTemplateData.skip_tags) ? jobTemplateData.skip_tags.split(',') + .map((i) => ({name: i, label: i, value: i})) : []; + $scope.skip_tags = $scope.skip_tag_options; + master.skip_tags = $scope.skip_tags; + + $scope.ask_job_type_on_launch = (jobTemplateData.ask_job_type_on_launch) ? true : false; + master.ask_job_type_on_launch = $scope.ask_job_type_on_launch; + + $scope.ask_inventory_on_launch = (jobTemplateData.ask_inventory_on_launch) ? true : false; + master.ask_inventory_on_launch = $scope.ask_inventory_on_launch; + + $scope.ask_credential_on_launch = (jobTemplateData.ask_credential_on_launch) ? true : false; + master.ask_credential_on_launch = $scope.ask_credential_on_launch; + + if (jobTemplateData.host_config_key) { + $scope.example_config_key = jobTemplateData.host_config_key; + } + $scope.example_template_id = id; + $scope.setCallbackHelp(); + + $scope.callback_url = $scope.callback_server_path + ((jobTemplateData.related.callback) ? jobTemplateData.related.callback : + GetBasePath('job_templates') + id + '/callback/'); + master.callback_url = $scope.callback_url; + + $scope.can_edit = jobTemplateData.summary_fields.user_capabilities.edit; + + if($scope.job_template_obj.summary_fields.user_capabilities.edit) { + MultiCredentialService.loadCredentials(jobTemplateData) + .then(([selectedCredentials, credTypes, credTypeOptions, + credTags]) => { + $scope.selectedCredentials = selectedCredentials; + $scope.credential_types = credTypes; + $scope.credentialTypeOptions = credTypeOptions; + $scope.credentialsToPost = credTags; + $scope.$emit('jobTemplateLoaded', master); + }); + } + else { + + if (jobTemplateData.summary_fields.credential) { + $scope.selectedCredentials.machine = jobTemplateData.summary_fields.credential; + } + + if (jobTemplateData.summary_fields.vault_credential) { + $scope.selectedCredentials.vault = jobTemplateData.summary_fields.vault_credential; + } + + // Extra credentials are not included in summary_fields so we have to go + // out and get them ourselves. + + let defers = [], + typesArray = [], + credTypeOptions; + + Rest.setUrl(jobTemplateData.related.extra_credentials); + defers.push(Rest.get() + .then((data) => { + $scope.selectedCredentials.extra = data.data.results; + }) + .catch(({data, status}) => { + ProcessErrors(null, data, status, null, + { + hdr: 'Error!', + msg: 'Failed to get extra credentials. ' + + 'Get returned status: ' + + status + }); + })); + + defers.push(MultiCredentialService.getCredentialTypes() + .then(({credential_types, credentialTypeOptions}) => { + typesArray = Object.keys(credential_types).map(key => credential_types[key]); + credTypeOptions = credentialTypeOptions; + }) + ); + + + return $q.all(defers).then(() => { + let machineAndVaultCreds = [], + extraCreds = []; + + if($scope.selectedCredentials.machine) { + machineAndVaultCreds.push($scope.selectedCredentials.machine); + } + if($scope.selectedCredentials.vault) { + machineAndVaultCreds.push($scope.selectedCredentials.vault); + } + + machineAndVaultCreds.map(cred => ({ + name: cred.name, + id: cred.id, + postType: cred.postType, + kind: typesArray + .filter(type => { + return cred.kind === type.kind || parseInt(cred.credential_type) === type.value; + })[0].name + ":" + })); + + extraCreds = extraCreds.concat($scope.selectedCredentials.extra).map(cred => ({ + name: cred.name, + id: cred.id, + postType: cred.postType, + kind: credTypeOptions + .filter(type => { + return parseInt(cred.credential_type) === type.value; + })[0].name + ":" + })); + + $scope.credentialsToPost = machineAndVaultCreds.concat(extraCreds); + + $scope.$emit('jobTemplateLoaded', master); + }); + + } }); if ($scope.removeChoicesReady) { diff --git a/awx/ui/client/src/templates/job_templates/factories/callback-help-init.factory.js b/awx/ui/client/src/templates/job_templates/factories/callback-help-init.factory.js index d0eada857c..e1c027a4e0 100644 --- a/awx/ui/client/src/templates/job_templates/factories/callback-help-init.factory.js +++ b/awx/ui/client/src/templates/job_templates/factories/callback-help-init.factory.js @@ -2,14 +2,7 @@ export default function CallbackHelpInit($q, $location, GetBasePath, Rest, JobTemplateForm, GenerateForm, $stateParams, ProcessErrors, ParseVariableString, Empty, Wait, MultiCredentialService, $rootScope) { return function(params) { - var scope = params.scope, - defaultUrl = GetBasePath('job_templates'), - // generator = GenerateForm, - form = JobTemplateForm(), - // loadingFinishedCount = 0, - // base = $location.path().replace(/^\//, '').split('/')[0], - master = {}, - id = $stateParams.job_template_id; + var scope = params.scope; // checkSCMStatus, getPlaybooks, callback, // choicesCount = 0; @@ -44,211 +37,6 @@ export default scope.example_config_key = '5a8ec154832b780b9bdef1061764ae5a'; scope.example_template_id = 'N'; scope.setCallbackHelp(); - - // this fills the job template form both on copy of the job template - // and on edit - scope.fillJobTemplate = function(){ - // id = id || $rootScope.copy.id; - // Retrieve detail record and prepopulate the form - Rest.setUrl(defaultUrl + id); - Rest.get() - .success(function (data) { - scope.job_template_obj = data; - scope.name = data.name; - var fld, i; - for (fld in form.fields) { - if (fld !== 'variables' && fld !== 'survey' && fld !== 'forks' && data[fld] !== null && data[fld] !== undefined) { - if (form.fields[fld].type === 'select') { - if (scope[fld + '_options'] && scope[fld + '_options'].length > 0) { - for (i = 0; i < scope[fld + '_options'].length; i++) { - if (data[fld] === scope[fld + '_options'][i].value) { - scope[fld] = scope[fld + '_options'][i]; - } - } - } else { - scope[fld] = data[fld]; - } - } else { - scope[fld] = data[fld]; - if(!Empty(data.summary_fields.survey)) { - scope.survey_exists = true; - } - } - master[fld] = scope[fld]; - } - if (fld === 'forks') { - if (data[fld] !== 0) { - scope[fld] = data[fld]; - master[fld] = scope[fld]; - } - } - if (fld === 'variables') { - // Parse extra_vars, converting to YAML. - scope.variables = ParseVariableString(data.extra_vars); - master.variables = scope.variables; - } - if (form.fields[fld].type === 'lookup' && data.summary_fields[form.fields[fld].sourceModel]) { - scope[form.fields[fld].sourceModel + '_' + form.fields[fld].sourceField] = - data.summary_fields[form.fields[fld].sourceModel][form.fields[fld].sourceField]; - master[form.fields[fld].sourceModel + '_' + form.fields[fld].sourceField] = - scope[form.fields[fld].sourceModel + '_' + form.fields[fld].sourceField]; - } - if (form.fields[fld].type === 'checkbox_group') { - for(var j=0; j ({name: i, label: i, value: i})) : []; - scope.job_tags = scope.job_tag_options; - master.job_tags = scope.job_tags; - - scope.skip_tag_options = (data.skip_tags) ? data.skip_tags.split(',') - .map((i) => ({name: i, label: i, value: i})) : []; - scope.skip_tags = scope.skip_tag_options; - master.skip_tags = scope.skip_tags; - - scope.ask_job_type_on_launch = (data.ask_job_type_on_launch) ? true : false; - master.ask_job_type_on_launch = scope.ask_job_type_on_launch; - - scope.ask_inventory_on_launch = (data.ask_inventory_on_launch) ? true : false; - master.ask_inventory_on_launch = scope.ask_inventory_on_launch; - - scope.ask_credential_on_launch = (data.ask_credential_on_launch) ? true : false; - master.ask_credential_on_launch = scope.ask_credential_on_launch; - - if (data.host_config_key) { - scope.example_config_key = data.host_config_key; - } - scope.example_template_id = id; - scope.setCallbackHelp(); - - scope.callback_url = scope.callback_server_path + ((data.related.callback) ? data.related.callback : - GetBasePath('job_templates') + id + '/callback/'); - master.callback_url = scope.callback_url; - - scope.can_edit = data.summary_fields.user_capabilities.edit; - - if(scope.job_template_obj.summary_fields.user_capabilities.edit) { - MultiCredentialService.loadCredentials(data) - .then(([selectedCredentials, credTypes, credTypeOptions, - credTags]) => { - scope.selectedCredentials = selectedCredentials; - scope.credential_types = credTypes; - scope.credentialTypeOptions = credTypeOptions; - scope.credentialsToPost = credTags; - scope.$emit('jobTemplateLoaded', master); - }); - } - else { - - if (data.summary_fields.credential) { - scope.selectedCredentials.machine = data.summary_fields.credential; - } - - if (data.summary_fields.vault_credential) { - scope.selectedCredentials.vault = data.summary_fields.vault_credential; - } - - // Extra credentials are not included in summary_fields so we have to go - // out and get them ourselves. - - let defers = [], - typesArray = [], - credTypeOptions; - - Rest.setUrl(data.related.extra_credentials); - defers.push(Rest.get() - .then((data) => { - scope.selectedCredentials.extra = data.data.results; - }) - .catch(({data, status}) => { - ProcessErrors(null, data, status, null, - { - hdr: 'Error!', - msg: 'Failed to get extra credentials. ' + - 'Get returned status: ' + - status - }); - })); - - defers.push(MultiCredentialService.getCredentialTypes() - .then(({credential_types, credentialTypeOptions}) => { - typesArray = Object.keys(credential_types).map(key => credential_types[key]); - credTypeOptions = credentialTypeOptions; - }) - ); - - - return $q.all(defers).then(() => { - let machineAndVaultCreds = [], - extraCreds = []; - - if(scope.selectedCredentials.machine) { - machineAndVaultCreds.push(scope.selectedCredentials.machine); - } - if(scope.selectedCredentials.vault) { - machineAndVaultCreds.push(scope.selectedCredentials.vault); - } - - machineAndVaultCreds.map(cred => ({ - name: cred.name, - id: cred.id, - postType: cred.postType, - kind: typesArray - .filter(type => { - return cred.kind === type.kind || parseInt(cred.credential_type) === type.value; - })[0].name + ":" - })); - - extraCreds = extraCreds.concat(scope.selectedCredentials.extra).map(cred => ({ - name: cred.name, - id: cred.id, - postType: cred.postType, - kind: credTypeOptions - .filter(type => { - return parseInt(cred.credential_type) === type.value; - })[0].name + ":" - })); - - scope.credentialsToPost = machineAndVaultCreds.concat(extraCreds); - - scope.$emit('jobTemplateLoaded', master); - }); - - } - }) - .error(function (data, status) { - ProcessErrors(scope, data, status, form, { - hdr: 'Error!', - msg: 'Failed to retrieve job template: ' + $stateParams.id + '. GET status: ' + status - }); - }); - }; }; } diff --git a/awx/ui/client/src/templates/job_templates/job-template.form.js b/awx/ui/client/src/templates/job_templates/job-template.form.js index 720bad0aad..ec7ec3a454 100644 --- a/awx/ui/client/src/templates/job_templates/job-template.form.js +++ b/awx/ui/client/src/templates/job_templates/job-template.form.js @@ -85,7 +85,7 @@ function(NotificationsList, CompletedJobsList, i18n) { ngChange: 'job_template_form.inventory_name.$validate()', text: i18n._('Prompt on launch') }, - ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate)' + ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || !canChangeInventory' }, project: { label: i18n._('Project'), @@ -100,13 +100,14 @@ function(NotificationsList, CompletedJobsList, i18n) { dataTitle: i18n._('Project'), dataPlacement: 'right', dataContainer: "body", - ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate)' + ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || !canChangeProject', + awLookupWhen: 'canChangeProject' }, playbook: { label: i18n._('Playbook'), type:'select', ngOptions: 'book for book in playbook_options track by book', - ngDisabled: "!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || disablePlaybookBecausePermissionDenied", + ngDisabled: "!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || !canChangeProject", id: 'playbook-select', required: true, column: 1, diff --git a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.directive.js b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.directive.js index 4155a44d39..6b9e8ba709 100644 --- a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.directive.js +++ b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.directive.js @@ -14,7 +14,7 @@ export default ['templateUrl', 'Rest', 'GetBasePath', 'generateList', '$compile' .then(kinds => { scope.credentialKinds = kinds; - scope.credentialKind = "" + kinds.Machine; + scope.credentialKind = scope.selectedCredentials.machine && scope.selectedCredentials.machine.readOnly ? (scope.selectedCredentials.vault && scope.selectedCredentials.vault.readOnly ? "" + kinds.Network : "" + kinds.Vault) : "" + kinds.Machine; scope.showModal = function() { $('#multi-credential-modal').modal('show'); @@ -50,6 +50,22 @@ export default ['templateUrl', 'Rest', 'GetBasePath', 'generateList', '$compile' .then(({credential_types, credentialTypeOptions}) => { scope.credential_types = credential_types; scope.credentialTypeOptions = credentialTypeOptions; + scope.allCredentialTypeOptions = _.cloneDeep(credentialTypeOptions); + + // We want to hide machine and vault dropdown options if a credential + // has already been selected for those types and the user interacting + // with the form doesn't have the ability to change them + for(let i=scope.credentialTypeOptions.length - 1; i >=0; i--) { + if((scope.selectedCredentials.machine && + scope.selectedCredentials.machine.credential_type_id === scope.credentialTypeOptions[i].value && + scope.selectedCredentials.machine.readOnly) || + (scope.selectedCredentials.vault && + scope.selectedCredentials.vault.credential_type_id === scope.credentialTypeOptions[i].value && + scope.selectedCredentials.vault.readOnly)) { + scope.credentialTypeOptions.splice(i, 1); + } + } + scope.$emit('multiCredentialModalLinked'); }); }); @@ -70,7 +86,7 @@ export default ['templateUrl', 'Rest', 'GetBasePath', 'generateList', '$compile' $scope.credTags = MultiCredentialService .updateCredentialTags($scope.selectedCredentials, - $scope.credentialTypeOptions); + $scope.allCredentialTypeOptions); }; let updateMachineCredentialList = function() { @@ -85,7 +101,7 @@ export default ['templateUrl', 'Rest', 'GetBasePath', 'generateList', '$compile' $scope.credTags = MultiCredentialService .updateCredentialTags($scope.selectedCredentials, - $scope.credentialTypeOptions); + $scope.allCredentialTypeOptions); }; let updateVaultCredentialList = function() { @@ -100,7 +116,7 @@ export default ['templateUrl', 'Rest', 'GetBasePath', 'generateList', '$compile' $scope.credTags = MultiCredentialService .updateCredentialTags($scope.selectedCredentials, - $scope.credentialTypeOptions); + $scope.allCredentialTypeOptions); }; let uncheckAllCredentials = function() { @@ -110,7 +126,7 @@ export default ['templateUrl', 'Rest', 'GetBasePath', 'generateList', '$compile' $scope.credTags = MultiCredentialService .updateCredentialTags($scope.selectedCredentials, - $scope.credentialTypeOptions); + $scope.allCredentialTypeOptions); }; let init = function() { diff --git a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.partial.html b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.partial.html index 22fa0699f4..f14933339e 100644 --- a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.partial.html +++ b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential-modal.partial.html @@ -27,13 +27,13 @@ class="MultiCredential-tagContainer ng-scope" ng-repeat="tag in credTags track by $index">
+ ng-click="removeCredential(tag.id)" + ng-if="!tag.readOnly">
-
+
diff --git a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.block.less b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.block.less index 86db1b3fff..4f007ea2f4 100644 --- a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.block.less +++ b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.block.less @@ -51,6 +51,10 @@ padding-left: 15px; } +.MultiCredential-tag--disabled { + background-color: @default-icon; +} + .MultiCredential-tag--deletable { margin-right: 0px; border-top-left-radius: 0px; diff --git a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.partial.html b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.partial.html index 6d51ae9ba1..912dc078a5 100644 --- a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.partial.html +++ b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.partial.html @@ -23,12 +23,12 @@ ng-repeat="tag in credentialsToPost track by $index">
+ ng-hide="fieldIsDisabled || tag.readOnly">
+ ng-class="{'MultiCredential-tag--deletable': !fieldIsDisabled && !tag.readOnly, 'MultiCredential-tag--disabled': tag.readOnly}"> {{ tag.kind }} diff --git a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js index 644097b451..9cdf912956 100644 --- a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js +++ b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js @@ -138,6 +138,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro name: cred.name, id: cred.id, postType: cred.postType, + readOnly: cred.readOnly, kind: typeOpts .filter(type => { return parseInt(cred.credential_type) === type.value; @@ -190,6 +191,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro /* User doesn't have read access to the machine credential, so use summary_fields */ selectedCredentials.machine = job_template_obj.summary_fields.credential; selectedCredentials.machine.credential_type = job_template_obj.summary_fields.credential.credential_type_id; + selectedCredentials.machine.readOnly = true; } else { ProcessErrors( null, data, status, null, @@ -214,6 +216,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro /* User doesn't have read access to the vault credential, so use summary_fields */ selectedCredentials.vault = job_template_obj.summary_fields.vault_credential; selectedCredentials.vault.credential_type = job_template_obj.summary_fields.vault_credential.credential_type_id; + selectedCredentials.vault.readOnly = true; } else { ProcessErrors( null, data, status, null, @@ -240,6 +243,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro selectedCredentials.extra = job_template_obj.summary_fields.extra_credentials; _.map(selectedCredentials.extra, (cred) => { cred.credential_type = cred.credential_type_id; + cred.readOnly = true; return cred; }); } else { diff --git a/awx/ui/client/src/templates/main.js b/awx/ui/client/src/templates/main.js index dbaf88d3e6..cd578d5d21 100644 --- a/awx/ui/client/src/templates/main.js +++ b/awx/ui/client/src/templates/main.js @@ -137,6 +137,61 @@ angular.module('templates', [surveyMaker.name, templatesList.name, jobTemplates. }, resolve: { edit: { + jobTemplateData: ['$stateParams', 'TemplatesService', 'ProcessErrors', + function($stateParams, TemplatesService, ProcessErrors) { + return TemplatesService.getJobTemplate($stateParams.job_template_id) + .then(function(res) { + return res.data; + }).catch(function(response){ + ProcessErrors(null, response.data, response.status, null, { + hdr: 'Error!', + msg: 'Failed to get job template. GET returned status: ' + + response.status + }); + }); + }], + canChangeProject: ['Rest', 'ProcessErrors', 'jobTemplateData', + function(Rest, ProcessErrors, jobTemplateData) { + Rest.setUrl(jobTemplateData.related.project); + return Rest.get() + .then(() => { + return true; + }) + .catch(({data, status}) => { + if (status === 403) { + /* User doesn't have read access to the project, no problem. */ + } else { + ProcessErrors(null, data, status, null, { + hdr: 'Error!', + msg: 'Failed to get project. GET returned ' + + 'status: ' + status + }); + } + + return false; + }); + }], + canChangeInventory: ['Rest', 'ProcessErrors', 'jobTemplateData', + function(Rest, ProcessErrors, jobTemplateData) { + Rest.setUrl(jobTemplateData.related.inventory); + return Rest.get() + .then(() => { + return true; + }) + .catch(({data, status}) => { + if (status === 403) { + /* User doesn't have read access to the project, no problem. */ + } else { + ProcessErrors(null, data, status, null, { + hdr: 'Error!', + msg: 'Failed to get project. GET returned ' + + 'status: ' + status + }); + } + + return false; + }); + }], InstanceGroupsData: ['$stateParams', 'Rest', 'GetBasePath', 'ProcessErrors', function($stateParams, Rest, GetBasePath, ProcessErrors){ let path = `${GetBasePath('job_templates')}${$stateParams.job_template_id}/instance_groups/`; @@ -155,32 +210,32 @@ angular.module('templates', [surveyMaker.name, templatesList.name, jobTemplates. }); }); }], - availableLabels: ['Rest', '$stateParams', 'GetBasePath', 'ProcessErrors', 'TemplatesService', - function(Rest, $stateParams, GetBasePath, ProcessErrors, TemplatesService) { - return TemplatesService.getAllLabelOptions() - .then(function(labels){ - return labels; - }).catch(function(response){ - ProcessErrors(null, response.data, response.status, null, { - hdr: 'Error!', - msg: 'Failed to get labels. GET returned status: ' + - response.status - }); + availableLabels: ['Rest', '$stateParams', 'GetBasePath', 'ProcessErrors', 'TemplatesService', + function(Rest, $stateParams, GetBasePath, ProcessErrors, TemplatesService) { + return TemplatesService.getAllLabelOptions() + .then(function(labels){ + return labels; + }).catch(function(response){ + ProcessErrors(null, response.data, response.status, null, { + hdr: 'Error!', + msg: 'Failed to get labels. GET returned status: ' + + response.status }); - }], - selectedLabels: ['Rest', '$stateParams', 'GetBasePath', 'TemplatesService', 'ProcessErrors', - function(Rest, $stateParams, GetBasePath, TemplatesService, ProcessErrors) { - return TemplatesService.getAllJobTemplateLabels($stateParams.job_template_id) - .then(function(labels){ - return labels; - }).catch(function(response){ - ProcessErrors(null, response.data, response.status, null, { - hdr: 'Error!', - msg: 'Failed to get workflow job template labels. GET returned status: ' + - response.status - }); + }); + }], + selectedLabels: ['Rest', '$stateParams', 'GetBasePath', 'TemplatesService', 'ProcessErrors', + function(Rest, $stateParams, GetBasePath, TemplatesService, ProcessErrors) { + return TemplatesService.getAllJobTemplateLabels($stateParams.job_template_id) + .then(function(labels){ + return labels; + }).catch(function(response){ + ProcessErrors(null, response.data, response.status, null, { + hdr: 'Error!', + msg: 'Failed to get workflow job template labels. GET returned status: ' + + response.status }); - }] + }); + }] } } }); From d3df5de0ced929deb0e24099a70de8b47a0b2f04 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 13 Sep 2017 13:28:26 -0400 Subject: [PATCH 26/52] build extra_cred related urls for jobs and JTs a less volatile way see: https://github.com/ansible/ansible-tower/issues/7635 --- awx/api/serializers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 5dfb375133..873f822171 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2334,8 +2334,13 @@ class JobOptionsSerializer(LabelsListMixin, BaseSerializer): if obj.vault_credential: res['vault_credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.vault_credential.pk}) if self.version > 1: - view = 'api:%s_extra_credentials_list' % camelcase_to_underscore(obj.__class__.__name__) - res['extra_credentials'] = self.reverse(view, kwargs={'pk': obj.pk}) + if isinstance(obj, UnifiedJobTemplate): + res['extra_credentials'] = self.reverse( + 'api:job_template_extra_credentials_list', + kwargs={'pk': obj.pk} + ) + elif isinstance(obj, UnifiedJob): + res['extra_credentials'] = self.reverse('api:job_extra_credentials_list', kwargs={'pk': obj.pk}) else: cloud_cred = obj.cloud_credential if cloud_cred: From 1ce3c7937b9ac2454f6dece846a3132f8ce83ffb Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 13 Sep 2017 13:34:39 -0400 Subject: [PATCH 27/52] use request response cache for license checks --- awx/conf/license.py | 3 ++- awx/main/utils/common.py | 5 +++-- awx/settings/defaults.py | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/awx/conf/license.py b/awx/conf/license.py index 57456f90fa..17a1d323c7 100644 --- a/awx/conf/license.py +++ b/awx/conf/license.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework.exceptions import APIException # Tower -from awx.main.utils.common import get_licenser +from awx.main.utils.common import get_licenser, memoize __all__ = ['LicenseForbids', 'get_license', 'get_licensed_features', 'feature_enabled', 'feature_exists'] @@ -40,6 +40,7 @@ def get_licensed_features(): return features +@memoize(cache_name='ephemeral') def feature_enabled(name): """Return True if the requested feature is enabled, False otherwise.""" validated_license_data = _get_validated_license_data() diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 58d795567f..1c1f2890f8 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -107,13 +107,14 @@ class RequireDebugTrueOrTest(logging.Filter): return settings.DEBUG or 'test' in sys.argv -def memoize(ttl=60, cache_key=None): +def memoize(ttl=60, cache_key=None, cache_name='default'): ''' Decorator to wrap a function and cache its result. ''' - from django.core.cache import cache + from django.core.cache import caches def _memoizer(f, *args, **kwargs): + cache = caches[cache_name] key = cache_key or slugify('%s %r %r' % (f.__name__, args, kwargs)) value = cache.get(key) if value is None: diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index c4af5d8d10..514b6ac731 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -481,6 +481,9 @@ if is_testing(): 'default': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', }, + 'ephemeral': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, } else: CACHES = { @@ -488,6 +491,9 @@ else: 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': 'memcached:11211', }, + 'ephemeral': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, } # Social Auth configuration. From f90771ee1a3ba39feefd47676e01ecc139d581e8 Mon Sep 17 00:00:00 2001 From: mabashian Date: Wed, 13 Sep 2017 16:48:04 -0400 Subject: [PATCH 28/52] More fixes for JT admin edit use cases Signed-off-by: mabashian --- .../add-job-template/job-template-add.controller.js | 3 +-- .../edit-job-template/job-template-edit.controller.js | 9 ++++----- .../src/templates/job_templates/job-template.form.js | 8 ++++---- .../multi-credential/multi-credential.service.js | 7 ++++++- awx/ui/client/src/templates/main.js | 4 ++-- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js b/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js index 1438f89f37..9cd73e1da7 100644 --- a/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js +++ b/awx/ui/client/src/templates/job_templates/add-job-template/job-template-add.controller.js @@ -37,8 +37,7 @@ $scope.mode = "add"; $scope.parseType = 'yaml'; $scope.credentialNotPresent = false; - $scope.canChangeProject = true; - $scope.canChangeInventory = true; + $scope.canGetAllRelatedResources = true; md5Setup({ scope: $scope, diff --git a/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js b/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js index 5e38774766..b25905e7bf 100644 --- a/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js +++ b/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js @@ -18,7 +18,7 @@ export default 'Empty', 'Prompt', 'ToJSON', 'GetChoices', 'CallbackHelpInit', 'InitiatePlaybookRun' , 'initSurvey', '$state', 'CreateSelect2', 'ToggleNotification','$q', 'InstanceGroupsService', 'InstanceGroupsData', 'MultiCredentialService', 'availableLabels', - 'canChangeProject', 'canChangeInventory', 'jobTemplateData', 'ParseVariableString', + 'canGetProject', 'canGetInventory', 'jobTemplateData', 'ParseVariableString', function( $filter, $scope, $rootScope, $location, $stateParams, JobTemplateForm, GenerateForm, Rest, Alert, @@ -26,7 +26,7 @@ export default ParseTypeChange, Wait, selectedLabels, i18n, Empty, Prompt, ToJSON, GetChoices, CallbackHelpInit, InitiatePlaybookRun, SurveyControllerInit, $state, CreateSelect2, ToggleNotification, $q, InstanceGroupsService, InstanceGroupsData, MultiCredentialService, availableLabels, - canChangeProject, canChangeInventory, jobTemplateData, ParseVariableString + canGetProject, canGetInventory, jobTemplateData, ParseVariableString ) { $scope.$watch('job_template_obj.summary_fields.user_capabilities.edit', function(val) { @@ -60,8 +60,6 @@ export default $scope.surveyTooltip = i18n._('Surveys allow users to be prompted at job launch with a series of questions related to the job. This allows for variables to be defined that affect the playbook run at time of launch.'); $scope.job_tag_options = []; $scope.skip_tag_options = []; - $scope.canChangeProject = canChangeProject; - $scope.canChangeInventory = canChangeInventory; SurveyControllerInit({ scope: $scope, @@ -361,7 +359,8 @@ export default if($scope.job_template_obj.summary_fields.user_capabilities.edit) { MultiCredentialService.loadCredentials(jobTemplateData) .then(([selectedCredentials, credTypes, credTypeOptions, - credTags]) => { + credTags, credentialGetPermissionDenied]) => { + $scope.canGetAllRelatedResources = canGetProject && canGetInventory && !credentialGetPermissionDenied ? true : false; $scope.selectedCredentials = selectedCredentials; $scope.credential_types = credTypes; $scope.credentialTypeOptions = credTypeOptions; diff --git a/awx/ui/client/src/templates/job_templates/job-template.form.js b/awx/ui/client/src/templates/job_templates/job-template.form.js index ec7ec3a454..891eb72dab 100644 --- a/awx/ui/client/src/templates/job_templates/job-template.form.js +++ b/awx/ui/client/src/templates/job_templates/job-template.form.js @@ -85,7 +85,7 @@ function(NotificationsList, CompletedJobsList, i18n) { ngChange: 'job_template_form.inventory_name.$validate()', text: i18n._('Prompt on launch') }, - ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || !canChangeInventory' + ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || !canGetAllRelatedResources' }, project: { label: i18n._('Project'), @@ -100,14 +100,14 @@ function(NotificationsList, CompletedJobsList, i18n) { dataTitle: i18n._('Project'), dataPlacement: 'right', dataContainer: "body", - ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || !canChangeProject', - awLookupWhen: 'canChangeProject' + ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || !canGetAllRelatedResources', + awLookupWhen: 'canGetAllRelatedResources' }, playbook: { label: i18n._('Playbook'), type:'select', ngOptions: 'book for book in playbook_options track by book', - ngDisabled: "!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || !canChangeProject", + ngDisabled: "!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate) || !canGetAllRelatedResources", id: 'playbook-select', required: true, column: 1, diff --git a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js index 9cdf912956..806a8bb442 100644 --- a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js +++ b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js @@ -179,6 +179,8 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro let credDefers = []; let job_template_obj = data; + let credentialGetPermissionDenied = false; + // get machine credential if (data.related.credential) { Rest.setUrl(data.related.credential); @@ -189,6 +191,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro .catch(({data, status}) => { if (status === 403) { /* User doesn't have read access to the machine credential, so use summary_fields */ + credentialGetPermissionDenied = true; selectedCredentials.machine = job_template_obj.summary_fields.credential; selectedCredentials.machine.credential_type = job_template_obj.summary_fields.credential.credential_type_id; selectedCredentials.machine.readOnly = true; @@ -214,6 +217,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro .catch(({data, status}) => { if (status === 403) { /* User doesn't have read access to the vault credential, so use summary_fields */ + credentialGetPermissionDenied = true; selectedCredentials.vault = job_template_obj.summary_fields.vault_credential; selectedCredentials.vault.credential_type = job_template_obj.summary_fields.vault_credential.credential_type_id; selectedCredentials.vault.readOnly = true; @@ -240,6 +244,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro .catch(({data, status}) => { if (status === 403) { /* User doesn't have read access to the extra credentials, so use summary_fields */ + credentialGetPermissionDenied = true; selectedCredentials.extra = job_template_obj.summary_fields.extra_credentials; _.map(selectedCredentials.extra, (cred) => { cred.credential_type = cred.credential_type_id; @@ -271,7 +276,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro .updateCredentialTags(selectedCredentials, credTypeOptions); return [selectedCredentials, credTypes, credTypeOptions, - credTags]; + credTags, credentialGetPermissionDenied]; }); }; diff --git a/awx/ui/client/src/templates/main.js b/awx/ui/client/src/templates/main.js index cd578d5d21..564309fffe 100644 --- a/awx/ui/client/src/templates/main.js +++ b/awx/ui/client/src/templates/main.js @@ -150,7 +150,7 @@ angular.module('templates', [surveyMaker.name, templatesList.name, jobTemplates. }); }); }], - canChangeProject: ['Rest', 'ProcessErrors', 'jobTemplateData', + canGetProject: ['Rest', 'ProcessErrors', 'jobTemplateData', function(Rest, ProcessErrors, jobTemplateData) { Rest.setUrl(jobTemplateData.related.project); return Rest.get() @@ -171,7 +171,7 @@ angular.module('templates', [surveyMaker.name, templatesList.name, jobTemplates. return false; }); }], - canChangeInventory: ['Rest', 'ProcessErrors', 'jobTemplateData', + canGetInventory: ['Rest', 'ProcessErrors', 'jobTemplateData', function(Rest, ProcessErrors, jobTemplateData) { Rest.setUrl(jobTemplateData.related.inventory); return Rest.get() From 9cf683ea75f2a672d6bc1b0b102078488f97a138 Mon Sep 17 00:00:00 2001 From: Jared Tabor Date: Wed, 13 Sep 2017 17:53:06 -0700 Subject: [PATCH 29/52] move socketPromise.resolve() until handshake message is received instead of happening after the socket.onopen event. --- awx/ui/client/src/shared/socket/socket.service.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/shared/socket/socket.service.js b/awx/ui/client/src/shared/socket/socket.service.js index d7fe283391..dec682a488 100644 --- a/awx/ui/client/src/shared/socket/socket.service.js +++ b/awx/ui/client/src/shared/socket/socket.service.js @@ -34,7 +34,6 @@ export default self.socket.onopen = function () { $log.debug("Websocket connection opened."); - socketPromise.resolve(); self.checkStatus(); if(needsResubscribing){ self.subscribe(self.getLast()); @@ -76,6 +75,10 @@ export default $log.debug('Received From Server: ' + e.data); var data = JSON.parse(e.data), str = ""; + if(_.has(data, "accept") && data.accept === true){ + socketPromise.resolve(); + return; + } if(data.group_name==="jobs" && !('status' in data)){ // we know that this must have been a // summary complete message b/c status is missing. From 549737405be4d85ecc56f0e3080d9708cda67b20 Mon Sep 17 00:00:00 2001 From: mabashian Date: Thu, 14 Sep 2017 09:14:27 -0400 Subject: [PATCH 30/52] Fixed multi credential service unit test failures Signed-off-by: mabashian --- .../multi-credential.service.js | 2 +- .../multi-credential.service-test.js | 78 ++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js index 806a8bb442..a22937fb61 100644 --- a/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js +++ b/awx/ui/client/src/templates/job_templates/multi-credential/multi-credential.service.js @@ -138,7 +138,7 @@ export default ['Rest', 'ProcessErrors', '$q', 'GetBasePath', function(Rest, Pro name: cred.name, id: cred.id, postType: cred.postType, - readOnly: cred.readOnly, + readOnly: cred.readOnly ? true : false, kind: typeOpts .filter(type => { return parseInt(cred.credential_type) === type.value; diff --git a/awx/ui/tests/spec/multi-credential/multi-credential.service-test.js b/awx/ui/tests/spec/multi-credential/multi-credential.service-test.js index 9f47abd231..98eeb48070 100644 --- a/awx/ui/tests/spec/multi-credential/multi-credential.service-test.js +++ b/awx/ui/tests/spec/multi-credential/multi-credential.service-test.js @@ -1,4 +1,4 @@ -'use strict' +'use strict'; describe('MultiCredentialService', () => { let MultiCredentialService; @@ -79,7 +79,7 @@ describe('MultiCredentialService', () => { expect(equal).toBe(true); }); - it('should return array of selected credentials (populated)', () => { + it('should return array of selected credentials (populated, not read only)', () => { let creds = { machine: { credential_type: 1, @@ -120,18 +120,92 @@ describe('MultiCredentialService', () => { name: 'ssh', id: 3, postType: 'machine', + readOnly: false, kind: 'SSH:' }, { name: 'aws', id: 4, postType: 'extra', + readOnly: false, kind: 'Amazon Web Services:' }, { name: 'gce', id: 5, postType: 'extra', + readOnly: false, + kind: 'Google Compute Engine:' + } + ]; + + let actual = MultiCredentialService + .updateCredentialTags(creds, typeOpts); + + let equal = _.isEqual(expected.sort(), actual.sort()); + + expect(equal).toBe(true); + }); + + it('should return array of selected credentials (populated, read only)', () => { + let creds = { + machine: { + credential_type: 1, + id: 3, + name: 'ssh', + readOnly: true + }, + extra: [ + { + credential_type: 2, + id: 4, + name: 'aws', + readOnly: true + }, + { + credential_type: 3, + id: 5, + name: 'gce', + readOnly: true + } + ] + }; + + let typeOpts = [ + { + name: 'SSH', + value: 1 + }, + { + name: 'Amazon Web Services', + value: 2 + }, + { + name: 'Google Compute Engine', + value: 3 + } + ]; + + let expected = [ + { + name: 'ssh', + id: 3, + postType: 'machine', + readOnly: true, + kind: 'SSH:' + }, + { + name: 'aws', + id: 4, + postType: 'extra', + readOnly: true, + kind: 'Amazon Web Services:' + }, + { + name: 'gce', + id: 5, + postType: 'extra', + readOnly: true, kind: 'Google Compute Engine:' } ]; From 39994186dfffd4d598d7de228200c28de5acfc22 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Thu, 14 Sep 2017 13:05:31 -0400 Subject: [PATCH 31/52] Improve script endpoint performance by removing forced ordering --- awx/api/views.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 0930e5e71b..5ac258eaef 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2384,24 +2384,24 @@ class InventoryScriptView(RetrieveAPIView): host = get_object_or_404(obj.hosts, name=hostname, **hosts_q) data = host.variables_dict else: - data = OrderedDict() + data = dict() if obj.variables_dict: - all_group = data.setdefault('all', OrderedDict()) + all_group = data.setdefault('all', dict()) all_group['vars'] = obj.variables_dict if obj.kind == 'smart': if len(obj.hosts.all()) == 0: return Response({}) else: - all_group = data.setdefault('all', OrderedDict()) - smart_hosts_qs = obj.hosts.all().order_by('name') + all_group = data.setdefault('all', dict()) + smart_hosts_qs = obj.hosts.all() smart_hosts = list(smart_hosts_qs.values_list('name', flat=True)) all_group['hosts'] = smart_hosts else: # Add hosts without a group to the all group. - groupless_hosts_qs = obj.hosts.filter(groups__isnull=True, **hosts_q).order_by('name') + groupless_hosts_qs = obj.hosts.filter(groups__isnull=True, **hosts_q) groupless_hosts = list(groupless_hosts_qs.values_list('name', flat=True)) if groupless_hosts: - all_group = data.setdefault('all', OrderedDict()) + all_group = data.setdefault('all', dict()) all_group['hosts'] = groupless_hosts # Build in-memory mapping of groups and their hosts. @@ -2409,7 +2409,6 @@ class InventoryScriptView(RetrieveAPIView): if 'enabled' in hosts_q: group_hosts_kw['host__enabled'] = hosts_q['enabled'] group_hosts_qs = Group.hosts.through.objects.filter(**group_hosts_kw) - group_hosts_qs = group_hosts_qs.order_by('host__name') group_hosts_qs = group_hosts_qs.values_list('group_id', 'host_id', 'host__name') group_hosts_map = {} for group_id, host_id, host_name in group_hosts_qs: @@ -2421,7 +2420,6 @@ class InventoryScriptView(RetrieveAPIView): from_group__inventory_id=obj.id, to_group__inventory_id=obj.id, ) - group_parents_qs = group_parents_qs.order_by('from_group__name') group_parents_qs = group_parents_qs.values_list('from_group_id', 'from_group__name', 'to_group_id') group_children_map = {} for from_group_id, from_group_name, to_group_id in group_parents_qs: @@ -2430,15 +2428,15 @@ class InventoryScriptView(RetrieveAPIView): # Now use in-memory maps to build up group info. for group in obj.groups.all(): - group_info = OrderedDict() + group_info = dict() group_info['hosts'] = group_hosts_map.get(group.id, []) group_info['children'] = group_children_map.get(group.id, []) group_info['vars'] = group.variables_dict data[group.name] = group_info if hostvars: - data.setdefault('_meta', OrderedDict()) - data['_meta'].setdefault('hostvars', OrderedDict()) + data.setdefault('_meta', dict()) + data['_meta'].setdefault('hostvars', dict()) for host in obj.hosts.filter(**hosts_q): data['_meta']['hostvars'][host.name] = host.variables_dict From 8c2b9905d1cefe3fb605e027a4642ed2b8a4da9b Mon Sep 17 00:00:00 2001 From: John Mitchell Date: Thu, 14 Sep 2017 13:59:58 -0400 Subject: [PATCH 32/52] fix config forms from having save button disabled when no license is given --- .../src/configuration/auth-form/sub-forms/auth-ldap.form.js | 2 +- .../configuration/auth-form/sub-forms/auth-radius.form.js | 2 +- .../src/configuration/auth-form/sub-forms/auth-saml.form.js | 2 +- .../configuration/auth-form/sub-forms/auth-tacacs.form.js | 2 +- awx/ui/client/src/configuration/configuration.controller.js | 6 +++++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/awx/ui/client/src/configuration/auth-form/sub-forms/auth-ldap.form.js b/awx/ui/client/src/configuration/auth-form/sub-forms/auth-ldap.form.js index 6bd55b801f..c4fa3c1a6b 100644 --- a/awx/ui/client/src/configuration/auth-form/sub-forms/auth-ldap.form.js +++ b/awx/ui/client/src/configuration/auth-form/sub-forms/auth-ldap.form.js @@ -100,7 +100,7 @@ export default ['i18n', function(i18n) { }, save: { ngClick: 'vm.formSave()', - ngDisabled: "license_type !== 'enterprise' || form.$invalid || form.$pending" + ngDisabled: "license_type !== 'enterprise' && license_type !== 'open' || configuration_ldap_template_form.$invalid || configuration_ldap_template_form.$pending" } } }; diff --git a/awx/ui/client/src/configuration/auth-form/sub-forms/auth-radius.form.js b/awx/ui/client/src/configuration/auth-form/sub-forms/auth-radius.form.js index 333d5be2c3..c1d6873303 100644 --- a/awx/ui/client/src/configuration/auth-form/sub-forms/auth-radius.form.js +++ b/awx/ui/client/src/configuration/auth-form/sub-forms/auth-radius.form.js @@ -39,7 +39,7 @@ export default ['i18n', function(i18n) { }, save: { ngClick: 'vm.formSave()', - ngDisabled: "license_type !== 'enterprise' || form.$invalid || form.$pending" + ngDisabled: "license_type !== 'enterprise' && license_type !== 'open' || configuration_radius_template_form.$invalid || configuration_radius_template_form.$pending" } } }; diff --git a/awx/ui/client/src/configuration/auth-form/sub-forms/auth-saml.form.js b/awx/ui/client/src/configuration/auth-form/sub-forms/auth-saml.form.js index 08dd556ba3..dd3ed43708 100644 --- a/awx/ui/client/src/configuration/auth-form/sub-forms/auth-saml.form.js +++ b/awx/ui/client/src/configuration/auth-form/sub-forms/auth-saml.form.js @@ -92,7 +92,7 @@ export default ['i18n', function(i18n) { }, save: { ngClick: 'vm.formSave()', - ngDisabled: "license_type !== 'enterprise' || form.$invalid || form.$pending" + ngDisabled: "license_type !== 'enterprise' && license_type !== 'open' || configuration_saml_template_form.$invalid || configuration_saml_template_form.$pending" } } }; diff --git a/awx/ui/client/src/configuration/auth-form/sub-forms/auth-tacacs.form.js b/awx/ui/client/src/configuration/auth-form/sub-forms/auth-tacacs.form.js index d236b5f0cf..be235399c2 100644 --- a/awx/ui/client/src/configuration/auth-form/sub-forms/auth-tacacs.form.js +++ b/awx/ui/client/src/configuration/auth-form/sub-forms/auth-tacacs.form.js @@ -52,7 +52,7 @@ export default ['i18n', function(i18n) { }, save: { ngClick: 'vm.formSave()', - ngDisabled: "license_type !== 'enterprise' || form.$invalid || form.$pending" + ngDisabled: "license_type !== 'enterprise' && license_type !== 'open' || configuration_tacacs_template_form.$invalid || configuration_tacacs_template_form.$pending" } } }; diff --git a/awx/ui/client/src/configuration/configuration.controller.js b/awx/ui/client/src/configuration/configuration.controller.js index 72cae0f2d2..465c89890e 100644 --- a/awx/ui/client/src/configuration/configuration.controller.js +++ b/awx/ui/client/src/configuration/configuration.controller.js @@ -95,7 +95,11 @@ export default [ } else { if (key === "LICENSE") { - $scope.license_type = data[key].license_type; + if (_.isEmpty(data[key])) { + $scope.license_type = "open"; + } else { + $scope.license_type = data[key].license_type; + } } //handle nested objects if(ConfigurationUtils.isEmpty(data[key])) { From 57c9224b5c5cf0635d84da7816360279cdf1c8aa Mon Sep 17 00:00:00 2001 From: gconsidine Date: Thu, 14 Sep 2017 15:31:29 -0400 Subject: [PATCH 33/52] Fix host filter parsing --- .../smart-search/smart-search.controller.js | 2 +- .../shared/smart-search/smart-search.service.js | 15 +++++++++++---- .../smart-search/smart-search.service-test.js | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/awx/ui/client/src/shared/smart-search/smart-search.controller.js b/awx/ui/client/src/shared/smart-search/smart-search.controller.js index c3898dac41..6f5488ad1d 100644 --- a/awx/ui/client/src/shared/smart-search/smart-search.controller.js +++ b/awx/ui/client/src/shared/smart-search/smart-search.controller.js @@ -164,7 +164,7 @@ export default ['$stateParams', '$scope', '$state', 'GetBasePath', 'QuerySet', ' let splitTerms; if ($scope.singleSearchParam === 'host_filter') { - splitTerms = SmartSearchService.splitHostIntoTerms(terms); + splitTerms = SmartSearchService.splitFilterIntoTerms(terms); } else { splitTerms = SmartSearchService.splitSearchIntoTerms(terms); } diff --git a/awx/ui/client/src/shared/smart-search/smart-search.service.js b/awx/ui/client/src/shared/smart-search/smart-search.service.js index 1fea6ab4c6..3e97ec45e3 100644 --- a/awx/ui/client/src/shared/smart-search/smart-search.service.js +++ b/awx/ui/client/src/shared/smart-search/smart-search.service.js @@ -6,10 +6,14 @@ export default [function() { * work is done to encode quotes in quoted values and the spaces within those quoted * values before calling to `splitSearchIntoTerms`. */ - splitHostIntoTerms (searchString) { + splitFilterIntoTerms (searchString) { let groups = []; let quoted; + if (!searchString.includes(' ')) { + return this.splitSearchIntoTerms(this.encode(searchString)); + } + searchString.split(' ').forEach(substring => { if (substring.includes(':"')) { quoted = substring; @@ -17,9 +21,7 @@ export default [function() { quoted += ` ${substring}`; if (substring.includes('"')) { - quoted = quoted.replace(/"/g, encodeURIComponent('"')); - quoted = quoted.replace(/ /g, encodeURIComponent(' ')); - groups.push(quoted); + groups.push(this.encode(quoted)); quoted = undefined; } } else { @@ -29,6 +31,11 @@ export default [function() { return this.splitSearchIntoTerms(groups.join(' ')); }, + encode (string) { + string = string.replace(/'/g, '%27'); + + return string.replace(/("| )/g, match => encodeURIComponent(match)); + }, splitSearchIntoTerms(searchString) { return searchString.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g); }, diff --git a/awx/ui/tests/spec/smart-search/smart-search.service-test.js b/awx/ui/tests/spec/smart-search/smart-search.service-test.js index 13af06e51c..a6441719f1 100644 --- a/awx/ui/tests/spec/smart-search/smart-search.service-test.js +++ b/awx/ui/tests/spec/smart-search/smart-search.service-test.js @@ -42,4 +42,19 @@ describe('Service: SmartSearch', () => { }); }); + describe('fn splitFilterIntoTerms', () => { + it('should convert the filter term to a key and value with encode quotes and spaces', () => { + expect(SmartSearchService.splitFilterIntoTerms('foo')).toEqual(["foo"]); + expect(SmartSearchService.splitFilterIntoTerms('foo bar')).toEqual(["foo", "bar"]); + expect(SmartSearchService.splitFilterIntoTerms('name:foo bar')).toEqual(["name:foo", "bar"]); + expect(SmartSearchService.splitFilterIntoTerms('name:foo description:bar')).toEqual(["name:foo", "description:bar"]); + expect(SmartSearchService.splitFilterIntoTerms('name:"foo bar"')).toEqual(["name:%22foo%20bar%22"]); + expect(SmartSearchService.splitFilterIntoTerms('name:"foo bar" description:"bar foo"')).toEqual(["name:%22foo%20bar%22", "description:%22bar%20foo%22"]); + expect(SmartSearchService.splitFilterIntoTerms('name:"foo bar" a b c')).toEqual(["name:%22foo%20bar%22", 'a', 'b', 'c']); + expect(SmartSearchService.splitFilterIntoTerms('name:"1"')).toEqual(["name:%221%22"]); + expect(SmartSearchService.splitFilterIntoTerms('name:1')).toEqual(["name:1"]); + expect(SmartSearchService.splitFilterIntoTerms(`name:"foo ba'r" a b c`)).toEqual(["name:%22foo%20ba%27r%22", 'a', 'b', 'c']); + }); + }); + }); From eab3cb8efdd95c40050e44ffa098d7d5aa49fe8c Mon Sep 17 00:00:00 2001 From: gconsidine Date: Thu, 14 Sep 2017 15:57:52 -0400 Subject: [PATCH 34/52] Add support for quoted without spaces and falsey input --- .../src/shared/smart-search/smart-search.service.js | 10 +++++++++- .../spec/smart-search/smart-search.service-test.js | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/shared/smart-search/smart-search.service.js b/awx/ui/client/src/shared/smart-search/smart-search.service.js index 3e97ec45e3..ffaa95bc8c 100644 --- a/awx/ui/client/src/shared/smart-search/smart-search.service.js +++ b/awx/ui/client/src/shared/smart-search/smart-search.service.js @@ -7,6 +7,10 @@ export default [function() { * values before calling to `splitSearchIntoTerms`. */ splitFilterIntoTerms (searchString) { + if (!searchString) { + return null; + } + let groups = []; let quoted; @@ -16,7 +20,11 @@ export default [function() { searchString.split(' ').forEach(substring => { if (substring.includes(':"')) { - quoted = substring; + if (/"$/.test(substring)) { + groups.push(this.encode(substring)); + } else { + quoted = substring; + } } else if (quoted) { quoted += ` ${substring}`; diff --git a/awx/ui/tests/spec/smart-search/smart-search.service-test.js b/awx/ui/tests/spec/smart-search/smart-search.service-test.js index a6441719f1..6c2ad05f09 100644 --- a/awx/ui/tests/spec/smart-search/smart-search.service-test.js +++ b/awx/ui/tests/spec/smart-search/smart-search.service-test.js @@ -44,6 +44,7 @@ describe('Service: SmartSearch', () => { describe('fn splitFilterIntoTerms', () => { it('should convert the filter term to a key and value with encode quotes and spaces', () => { + expect(SmartSearchService.splitFilterIntoTerms()).toEqual(null); expect(SmartSearchService.splitFilterIntoTerms('foo')).toEqual(["foo"]); expect(SmartSearchService.splitFilterIntoTerms('foo bar')).toEqual(["foo", "bar"]); expect(SmartSearchService.splitFilterIntoTerms('name:foo bar')).toEqual(["name:foo", "bar"]); @@ -54,6 +55,8 @@ describe('Service: SmartSearch', () => { expect(SmartSearchService.splitFilterIntoTerms('name:"1"')).toEqual(["name:%221%22"]); expect(SmartSearchService.splitFilterIntoTerms('name:1')).toEqual(["name:1"]); expect(SmartSearchService.splitFilterIntoTerms(`name:"foo ba'r" a b c`)).toEqual(["name:%22foo%20ba%27r%22", 'a', 'b', 'c']); + expect(SmartSearchService.splitFilterIntoTerms('name:"foobar" other:"barbaz"')).toEqual(["name:%22foobar%22", "other:%22barbaz%22"]); + expect(SmartSearchService.splitFilterIntoTerms('name:"foobar" other:"bar baz"')).toEqual(["name:%22foobar%22", "other:%22bar%20baz%22"]); }); }); From dd62e8ce921054ac1b4b48769b7d72ae524f3e97 Mon Sep 17 00:00:00 2001 From: gconsidine Date: Thu, 14 Sep 2017 17:27:27 -0400 Subject: [PATCH 35/52] Replace use of string.includes due lack of browser support --- .../src/shared/smart-search/smart-search.service.js | 8 ++------ .../tests/spec/smart-search/smart-search.service-test.js | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/awx/ui/client/src/shared/smart-search/smart-search.service.js b/awx/ui/client/src/shared/smart-search/smart-search.service.js index ffaa95bc8c..8e7b57271d 100644 --- a/awx/ui/client/src/shared/smart-search/smart-search.service.js +++ b/awx/ui/client/src/shared/smart-search/smart-search.service.js @@ -14,12 +14,8 @@ export default [function() { let groups = []; let quoted; - if (!searchString.includes(' ')) { - return this.splitSearchIntoTerms(this.encode(searchString)); - } - searchString.split(' ').forEach(substring => { - if (substring.includes(':"')) { + if (/:"/g.test(substring)) { if (/"$/.test(substring)) { groups.push(this.encode(substring)); } else { @@ -28,7 +24,7 @@ export default [function() { } else if (quoted) { quoted += ` ${substring}`; - if (substring.includes('"')) { + if (/"/g.test(substring)) { groups.push(this.encode(quoted)); quoted = undefined; } diff --git a/awx/ui/tests/spec/smart-search/smart-search.service-test.js b/awx/ui/tests/spec/smart-search/smart-search.service-test.js index 6c2ad05f09..cd7aef4848 100644 --- a/awx/ui/tests/spec/smart-search/smart-search.service-test.js +++ b/awx/ui/tests/spec/smart-search/smart-search.service-test.js @@ -54,7 +54,7 @@ describe('Service: SmartSearch', () => { expect(SmartSearchService.splitFilterIntoTerms('name:"foo bar" a b c')).toEqual(["name:%22foo%20bar%22", 'a', 'b', 'c']); expect(SmartSearchService.splitFilterIntoTerms('name:"1"')).toEqual(["name:%221%22"]); expect(SmartSearchService.splitFilterIntoTerms('name:1')).toEqual(["name:1"]); - expect(SmartSearchService.splitFilterIntoTerms(`name:"foo ba'r" a b c`)).toEqual(["name:%22foo%20ba%27r%22", 'a', 'b', 'c']); + expect(SmartSearchService.splitFilterIntoTerms('name:"foo ba\'r" a b c')).toEqual(["name:%22foo%20ba%27r%22", 'a', 'b', 'c']); expect(SmartSearchService.splitFilterIntoTerms('name:"foobar" other:"barbaz"')).toEqual(["name:%22foobar%22", "other:%22barbaz%22"]); expect(SmartSearchService.splitFilterIntoTerms('name:"foobar" other:"bar baz"')).toEqual(["name:%22foobar%22", "other:%22bar%20baz%22"]); }); From bd95197709ede1b27e7134da8dd9d8e09f796f0b Mon Sep 17 00:00:00 2001 From: Jared Tabor Date: Thu, 14 Sep 2017 15:45:35 -0700 Subject: [PATCH 36/52] deleting socket on logout --- awx/ui/client/src/shared/socket/socket.service.js | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/ui/client/src/shared/socket/socket.service.js b/awx/ui/client/src/shared/socket/socket.service.js index dec682a488..39d67cbd56 100644 --- a/awx/ui/client/src/shared/socket/socket.service.js +++ b/awx/ui/client/src/shared/socket/socket.service.js @@ -119,6 +119,7 @@ export default disconnect: function(){ if(this.socket){ this.socket.close(); + delete this.socket; } }, subscribe: function(state){ From 7ccedfb1dfb5d2d3fee87ddda78c8590ebebdd45 Mon Sep 17 00:00:00 2001 From: Jared Tabor Date: Thu, 14 Sep 2017 15:50:35 -0700 Subject: [PATCH 37/52] adding console.log for debug purposes in prod, will delete later --- awx/ui/client/src/shared/socket/socket.service.js | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/ui/client/src/shared/socket/socket.service.js b/awx/ui/client/src/shared/socket/socket.service.js index 39d67cbd56..df05cdbede 100644 --- a/awx/ui/client/src/shared/socket/socket.service.js +++ b/awx/ui/client/src/shared/socket/socket.service.js @@ -120,6 +120,7 @@ export default if(this.socket){ this.socket.close(); delete this.socket; + console.log("Socket deleted: "+this.socket); } }, subscribe: function(state){ From 1bb6c17fe27d0deacc3bd0a3b763903fa35725df Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 13 Sep 2017 23:00:48 -0400 Subject: [PATCH 38/52] trick django-polymorphic into allowing `defer()` on polymorphic objects django-polymorphic itself generates queries for polymorphic object lookups, and these queries for UnifiedJob are *not* properly defering the `result_stdout_text` column, resulting in more very slow queries. This solution is _very_ hacky, and very specific to this specific version of Django and django-polymorphic, but it works until we can solve this problem the proper way in 3.3 (by removing large stdout blobs from the database). see: https://github.com/ansible/ansible-tower/issues/7568 --- awx/main/models/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 01bd68cc66..c044997fca 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -145,3 +145,15 @@ activity_stream_registrar.connect(WorkflowJob) # prevent API filtering on certain Django-supplied sensitive fields prevent_search(User._meta.get_field('password')) + +# Always, always, always defer result_stdout_text for polymorphic UnifiedJob rows +def defer_stdout(f): + def _wrapped(*args, **kwargs): + objs = f(*args, **kwargs) + objs.query.deferred_loading[0].add('result_stdout_text') + return objs + return _wrapped + + +for cls in UnifiedJob.__subclasses__(): + cls.base_objects.filter = defer_stdout(cls.base_objects.filter) From 7a958a1af1b7718f520d2a68cdab6370d6b64da8 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 14 Sep 2017 13:52:09 -0400 Subject: [PATCH 39/52] more result_stdout_text defer optimization for inventory updates see: https://github.com/ansible/ansible-tower/issues/7568 --- awx/api/views.py | 6 ++++++ awx/main/access.py | 1 + awx/main/models/__init__.py | 1 + 3 files changed, 8 insertions(+) diff --git a/awx/api/views.py b/awx/api/views.py index 0930e5e71b..c10a5654d2 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2670,6 +2670,12 @@ class InventoryUpdateList(ListAPIView): model = InventoryUpdate serializer_class = InventoryUpdateListSerializer + def get_queryset(self): + qs = super(InventoryUpdateList, self).get_queryset() + # TODO: remove this defer in 3.3 when we implement https://github.com/ansible/ansible-tower/issues/5436 + qs = qs.defer('result_stdout_text') + return qs + class InventoryUpdateDetail(UnifiedJobDeletionMixin, RetrieveDestroyAPIView): diff --git a/awx/main/access.py b/awx/main/access.py index 8297d2ba70..c2dae1397c 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -2084,6 +2084,7 @@ class UnifiedJobAccess(BaseAccess): # 'job_template__project', # 'job_template__credential', #) + # TODO: remove this defer in 3.3 when we implement https://github.com/ansible/ansible-tower/issues/5436 qs = qs.defer('result_stdout_text') return qs.all() diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index c044997fca..2443e16f45 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -147,6 +147,7 @@ activity_stream_registrar.connect(WorkflowJob) prevent_search(User._meta.get_field('password')) # Always, always, always defer result_stdout_text for polymorphic UnifiedJob rows +# TODO: remove this defer in 3.3 when we implement https://github.com/ansible/ansible-tower/issues/5436 def defer_stdout(f): def _wrapped(*args, **kwargs): objs = f(*args, **kwargs) From 645d2d852f1bdf5e68111a06fb9513a31682278c Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 15 Sep 2017 10:19:33 -0400 Subject: [PATCH 40/52] Update scan job migration to public project to use the correct URL --- awx/main/migrations/_scan_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/migrations/_scan_jobs.py b/awx/main/migrations/_scan_jobs.py index b953ba7d68..4dfc7cf972 100644 --- a/awx/main/migrations/_scan_jobs.py +++ b/awx/main/migrations/_scan_jobs.py @@ -14,7 +14,7 @@ def _create_fact_scan_project(ContentType, Project, org): ct = ContentType.objects.get_for_model(Project) name = "Tower Fact Scan - {}".format(org.name if org else "No Organization") proj = Project(name=name, - scm_url='https://github.com/ansible/tower-fact-modules', + scm_url='https://github.com/ansible/awx-facts-playbooks', scm_type='git', scm_update_on_launch=True, scm_update_cache_timeout=86400, From 47c976ee919933be852b683fb4b6df02421de68b Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 15 Sep 2017 11:14:10 -0400 Subject: [PATCH 41/52] Add a setting for vmware inventory cert validation This uses the environment var expected by the vmware inventory script. Defaults to off but can be turned on --- awx/main/tasks.py | 1 + awx/settings/defaults.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 1a6e0f25c1..6753fb8577 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1073,6 +1073,7 @@ class RunJob(BaseTask): env['VMWARE_USER'] = cloud_cred.username env['VMWARE_PASSWORD'] = decrypt_field(cloud_cred, 'password') env['VMWARE_HOST'] = cloud_cred.host + env['VMWARE_VALIDATE_CERTS'] = str(settings.VMWARE_VALIDATE_CERTS) elif cloud_cred and cloud_cred.kind == 'openstack': env['OS_CLIENT_CONFIG_FILE'] = cred_files.get(cloud_cred, '') diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 514b6ac731..437b3d6db8 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -722,7 +722,7 @@ VMWARE_GROUP_FILTER = r'^.+$' VMWARE_HOST_FILTER = r'^.+$' VMWARE_EXCLUDE_EMPTY_GROUPS = True - +VMWARE_VALIDATE_CERTS = False # --------------------------- # -- Google Compute Engine -- # --------------------------- From 8b7639fd3a142da1ef064b482f85b84a656d6636 Mon Sep 17 00:00:00 2001 From: mabashian Date: Fri, 15 Sep 2017 11:54:42 -0400 Subject: [PATCH 42/52] Removed extra GET call for extra creds when user cannot edit JT Signed-off-by: mabashian --- .../job-template-edit.controller.js | 94 ++++++++----------- 1 file changed, 37 insertions(+), 57 deletions(-) diff --git a/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js b/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js index b25905e7bf..2a74692075 100644 --- a/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js +++ b/awx/ui/client/src/templates/job_templates/edit-job-template/job-template-edit.controller.js @@ -378,71 +378,51 @@ export default $scope.selectedCredentials.vault = jobTemplateData.summary_fields.vault_credential; } - // Extra credentials are not included in summary_fields so we have to go - // out and get them ourselves. + if (jobTemplateData.summary_fields.extra_credentials) { + $scope.selectedCredentials.extra = jobTemplateData.summary_fields.extra_credentials; + } - let defers = [], - typesArray = [], - credTypeOptions; - - Rest.setUrl(jobTemplateData.related.extra_credentials); - defers.push(Rest.get() - .then((data) => { - $scope.selectedCredentials.extra = data.data.results; - }) - .catch(({data, status}) => { - ProcessErrors(null, data, status, null, - { - hdr: 'Error!', - msg: 'Failed to get extra credentials. ' + - 'Get returned status: ' + - status - }); - })); - - defers.push(MultiCredentialService.getCredentialTypes() + MultiCredentialService.getCredentialTypes() .then(({credential_types, credentialTypeOptions}) => { - typesArray = Object.keys(credential_types).map(key => credential_types[key]); - credTypeOptions = credentialTypeOptions; - }) - ); + let typesArray = Object.keys(credential_types).map(key => credential_types[key]); + let credTypeOptions = credentialTypeOptions; + let machineAndVaultCreds = [], + extraCreds = []; - return $q.all(defers).then(() => { - let machineAndVaultCreds = [], - extraCreds = []; + if($scope.selectedCredentials.machine) { + machineAndVaultCreds.push($scope.selectedCredentials.machine); + } + if($scope.selectedCredentials.vault) { + machineAndVaultCreds.push($scope.selectedCredentials.vault); + } - if($scope.selectedCredentials.machine) { - machineAndVaultCreds.push($scope.selectedCredentials.machine); - } - if($scope.selectedCredentials.vault) { - machineAndVaultCreds.push($scope.selectedCredentials.vault); - } + machineAndVaultCreds.map(cred => ({ + name: cred.name, + id: cred.id, + postType: cred.postType, + kind: typesArray + .filter(type => { + return cred.kind === type.kind || parseInt(cred.credential_type) === type.value; + })[0].name + ":" + })); - machineAndVaultCreds.map(cred => ({ - name: cred.name, - id: cred.id, - postType: cred.postType, - kind: typesArray - .filter(type => { - return cred.kind === type.kind || parseInt(cred.credential_type) === type.value; - })[0].name + ":" - })); + if($scope.selectedCredentials.extra && $scope.selectedCredentials.extra.length > 0) { + extraCreds = extraCreds.concat($scope.selectedCredentials.extra).map(cred => ({ + name: cred.name, + id: cred.id, + postType: cred.postType, + kind: credTypeOptions + .filter(type => { + return parseInt(cred.credential_type_id) === type.value; + })[0].name + ":" + })); + } - extraCreds = extraCreds.concat($scope.selectedCredentials.extra).map(cred => ({ - name: cred.name, - id: cred.id, - postType: cred.postType, - kind: credTypeOptions - .filter(type => { - return parseInt(cred.credential_type) === type.value; - })[0].name + ":" - })); + $scope.credentialsToPost = machineAndVaultCreds.concat(extraCreds); - $scope.credentialsToPost = machineAndVaultCreds.concat(extraCreds); - - $scope.$emit('jobTemplateLoaded', master); - }); + $scope.$emit('jobTemplateLoaded', master); + }); } }); From 325e7f3ccedb930879de5f7ce4cedd598f8b10ea Mon Sep 17 00:00:00 2001 From: Jared Tabor Date: Fri, 15 Sep 2017 10:37:35 -0700 Subject: [PATCH 43/52] checking socket.readyState before emitting socket message --- .../client/src/shared/socket/socket.service.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/awx/ui/client/src/shared/socket/socket.service.js b/awx/ui/client/src/shared/socket/socket.service.js index df05cdbede..d089060e02 100644 --- a/awx/ui/client/src/shared/socket/socket.service.js +++ b/awx/ui/client/src/shared/socket/socket.service.js @@ -191,14 +191,17 @@ export default var self = this; $log.debug('Sent to Websocket Server: ' + data); socketPromise.promise.then(function(){ - self.socket.send(data, function () { - var args = arguments; - self.scope.$apply(function () { - if (callback) { - callback.apply(self.socket, args); - } + console.log("socket readyState: " + self.socket.readyState); + if(self.socket.readyState === 1){ + self.socket.send(data, function () { + var args = arguments; + self.scope.$apply(function () { + if (callback) { + callback.apply(self.socket, args); + } + }); }); - }); + } }); }, addStateResolve: function(state, id){ From 9765623d0384b9704a0eacec310f069d753a0afd Mon Sep 17 00:00:00 2001 From: mabashian Date: Fri, 15 Sep 2017 13:55:44 -0400 Subject: [PATCH 44/52] Fixed search key styling with wider text Signed-off-by: mabashian --- .../src/shared/smart-search/smart-search.block.less | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/awx/ui/client/src/shared/smart-search/smart-search.block.less b/awx/ui/client/src/shared/smart-search/smart-search.block.less index d57ca5bbe9..cd97490440 100644 --- a/awx/ui/client/src/shared/smart-search/smart-search.block.less +++ b/awx/ui/client/src/shared/smart-search/smart-search.block.less @@ -32,7 +32,7 @@ .SmartSearch-searchTermContainer { flex: initial; - width: 100%; + flex-grow: 1; border: 1px solid @b7grey; border-radius: 4px; display: flex; @@ -167,7 +167,7 @@ color: @default-interface-txt; border: 1px solid @b7grey; cursor: pointer; - width: 70px; + min-width: 70px; height: 34px; line-height: 20px; } @@ -240,15 +240,6 @@ margin-right: 5px; } - -// Additional modal specific styles -.modal-body, #add-permissions-modal, -.JobResults { - .SmartSearch-searchTermContainer { - width: 100%; - } -} - @media (max-width: 700px) { .SmartSearch-bar { width: 100%; From dafd6acf1a2f49f771d1d4ee4fb76cf486363819 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Wed, 13 Sep 2017 23:00:41 -0400 Subject: [PATCH 45/52] Merge pull request #161 from wwitzel3/devel update social auth strategy to have fixes from social-app-django --- awx/settings/defaults.py | 2 +- awx/sso/strategies/__init__.py | 2 ++ awx/sso/strategies/django_strategy.py | 30 +++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 awx/sso/strategies/__init__.py create mode 100644 awx/sso/strategies/django_strategy.py diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 514b6ac731..023cbd8d07 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -497,7 +497,7 @@ else: } # Social Auth configuration. -SOCIAL_AUTH_STRATEGY = 'social.strategies.django_strategy.DjangoStrategy' +SOCIAL_AUTH_STRATEGY = 'awx.sso.strategies.django_strategy.AWXDjangoStrategy' SOCIAL_AUTH_STORAGE = 'social.apps.django_app.default.models.DjangoStorage' SOCIAL_AUTH_USER_MODEL = AUTH_USER_MODEL # noqa SOCIAL_AUTH_PIPELINE = ( diff --git a/awx/sso/strategies/__init__.py b/awx/sso/strategies/__init__.py new file mode 100644 index 0000000000..46176c348f --- /dev/null +++ b/awx/sso/strategies/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2017 Ansible, Inc. +# All Rights Reserved. diff --git a/awx/sso/strategies/django_strategy.py b/awx/sso/strategies/django_strategy.py new file mode 100644 index 0000000000..728e380dc4 --- /dev/null +++ b/awx/sso/strategies/django_strategy.py @@ -0,0 +1,30 @@ +# Copyright (c) 2017 Ansible, Inc. +# All Rights Reserved. + +from social.strategies.django_strategy import DjangoStrategy + + +class AWXDjangoStrategy(DjangoStrategy): + """A DjangoStrategy for python-social-auth containing + fixes and updates from social-app-django + + TODO: Revert back to using the default DjangoStrategy after + we upgrade to social-core / social-app-django. We will also + want to ensure we update the SOCIAL_AUTH_STRATEGY setting. + """ + + def __init__(self, storage, request=None, tpl=None): + super(AWXDjangoStrategy, self).__init__(storage, tpl) + + def request_port(self): + """Port in use for this request + https://github.com/python-social-auth/social-app-django/blob/master/social_django/strategy.py#L76 + """ + try: # django >= 1.9 + return self.request.get_port() + except AttributeError: # django < 1.9 + host_parts = self.request.get_host().split(':') + try: + return host_parts[1] + except IndexError: + return self.request.META['SERVER_PORT'] From f7312691e166fbdb242391bddfbeb80079e90d92 Mon Sep 17 00:00:00 2001 From: Jared Tabor Date: Fri, 15 Sep 2017 14:10:42 -0700 Subject: [PATCH 46/52] resubscribe to last group if readyState is 0, aka connection isn't open yet --- awx/ui/client/src/shared/socket/socket.service.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/awx/ui/client/src/shared/socket/socket.service.js b/awx/ui/client/src/shared/socket/socket.service.js index d089060e02..3508e742a0 100644 --- a/awx/ui/client/src/shared/socket/socket.service.js +++ b/awx/ui/client/src/shared/socket/socket.service.js @@ -34,6 +34,8 @@ export default self.socket.onopen = function () { $log.debug("Websocket connection opened."); + socketPromise.resolve(); + console.log('promise resolved, and readyState: '+ self.readyState); self.checkStatus(); if(needsResubscribing){ self.subscribe(self.getLast()); @@ -75,10 +77,6 @@ export default $log.debug('Received From Server: ' + e.data); var data = JSON.parse(e.data), str = ""; - if(_.has(data, "accept") && data.accept === true){ - socketPromise.resolve(); - return; - } if(data.group_name==="jobs" && !('status' in data)){ // we know that this must have been a // summary complete message b/c status is missing. @@ -191,7 +189,10 @@ export default var self = this; $log.debug('Sent to Websocket Server: ' + data); socketPromise.promise.then(function(){ - console.log("socket readyState: " + self.socket.readyState); + console.log("socket readyState at emit: " + self.socket.readyState); + if(self.socket.readyState === 0){ + self.subscribe(self.getLast()); + } if(self.socket.readyState === 1){ self.socket.send(data, function () { var args = arguments; From 4bbd48638923398fc99b4db612ec7c2f2c52f9d4 Mon Sep 17 00:00:00 2001 From: Jared Tabor Date: Sat, 16 Sep 2017 09:12:32 -0700 Subject: [PATCH 47/52] removing code that puts the UI in an infinite loop --- awx/ui/client/src/shared/socket/socket.service.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/awx/ui/client/src/shared/socket/socket.service.js b/awx/ui/client/src/shared/socket/socket.service.js index 3508e742a0..bb52260c87 100644 --- a/awx/ui/client/src/shared/socket/socket.service.js +++ b/awx/ui/client/src/shared/socket/socket.service.js @@ -190,9 +190,9 @@ export default $log.debug('Sent to Websocket Server: ' + data); socketPromise.promise.then(function(){ console.log("socket readyState at emit: " + self.socket.readyState); - if(self.socket.readyState === 0){ - self.subscribe(self.getLast()); - } + // if(self.socket.readyState === 0){ + // self.subscribe(self.getLast()); + // } if(self.socket.readyState === 1){ self.socket.send(data, function () { var args = arguments; From 8091b84344ccf11aed98a407830028544a4c5815 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Sat, 16 Sep 2017 09:57:52 -0400 Subject: [PATCH 48/52] Fixing up some activity stream deferreds --- awx/main/signals.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/awx/main/signals.py b/awx/main/signals.py index 81f6e17092..cf65401c0c 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -388,6 +388,9 @@ def activity_stream_create(sender, instance, created, **kwargs): # Skip recording any inventory source directly associated with a group. if isinstance(instance, InventorySource) and instance.deprecated_group: return + _type = type(instance) + if hasattr(_type, '_deferred', False): + return object1 = camelcase_to_underscore(instance.__class__.__name__) changes = model_to_dict(instance, model_serializer_mapping) # Special case where Job survey password variables need to be hidden @@ -421,6 +424,9 @@ def activity_stream_update(sender, instance, **kwargs): changes = model_instance_diff(old, new, model_serializer_mapping) if changes is None: return + _type = type(instance) + if hasattr(_type, '_deferred', False): + return object1 = camelcase_to_underscore(instance.__class__.__name__) activity_entry = ActivityStream( operation='update', @@ -445,6 +451,9 @@ def activity_stream_delete(sender, instance, **kwargs): # explicitly called with flag on in Inventory.schedule_deletion. if isinstance(instance, Inventory) and not kwargs.get('inventory_delete_flag', False): return + _type = type(instance) + if hasattr(_type, '_deferred', False): + return changes = model_to_dict(instance) object1 = camelcase_to_underscore(instance.__class__.__name__) activity_entry = ActivityStream( @@ -466,6 +475,9 @@ def activity_stream_associate(sender, instance, **kwargs): else: return obj1 = instance + _type = type(instance) + if hasattr(_type, '_deferred', False): + return object1=camelcase_to_underscore(obj1.__class__.__name__) obj_rel = sender.__module__ + "." + sender.__name__ @@ -476,6 +488,9 @@ def activity_stream_associate(sender, instance, **kwargs): if not obj2_actual.exists(): continue obj2_actual = obj2_actual[0] + _type = type(obj2_actual) + if hasattr(_type, '_deferred', False): + return if isinstance(obj2_actual, Role) and obj2_actual.content_object is not None: obj2_actual = obj2_actual.content_object object2 = camelcase_to_underscore(obj2_actual.__class__.__name__) From c07a4ff93d351a9b127f40745394c3b5b344fd71 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Sat, 16 Sep 2017 09:58:04 -0400 Subject: [PATCH 49/52] Catch any unhandled exceptions grabbing notification templates --- awx/main/tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 6753fb8577..459b296d5c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -316,7 +316,11 @@ def awx_periodic_scheduler(self): def _send_notification_templates(instance, status_str): if status_str not in ['succeeded', 'failed']: raise ValueError(_("status_str must be either succeeded or failed")) - notification_templates = instance.get_notification_templates() + try: + notification_templates = instance.get_notification_templates() + except: + logger.warn("No notification template defined for emitting notification") + notification_templates = None if notification_templates: if status_str == 'succeeded': notification_template_type = 'success' From ebbd42322d60865ed7b3c5eb7df871cb49be5063 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Sat, 16 Sep 2017 10:01:34 -0400 Subject: [PATCH 50/52] Fix an issue with hasattr call --- awx/main/signals.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/awx/main/signals.py b/awx/main/signals.py index cf65401c0c..998f28cc11 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -389,7 +389,7 @@ def activity_stream_create(sender, instance, created, **kwargs): if isinstance(instance, InventorySource) and instance.deprecated_group: return _type = type(instance) - if hasattr(_type, '_deferred', False): + if hasattr(_type, '_deferred'): return object1 = camelcase_to_underscore(instance.__class__.__name__) changes = model_to_dict(instance, model_serializer_mapping) @@ -425,7 +425,7 @@ def activity_stream_update(sender, instance, **kwargs): if changes is None: return _type = type(instance) - if hasattr(_type, '_deferred', False): + if hasattr(_type, '_deferred'): return object1 = camelcase_to_underscore(instance.__class__.__name__) activity_entry = ActivityStream( @@ -452,7 +452,7 @@ def activity_stream_delete(sender, instance, **kwargs): if isinstance(instance, Inventory) and not kwargs.get('inventory_delete_flag', False): return _type = type(instance) - if hasattr(_type, '_deferred', False): + if hasattr(_type, '_deferred'): return changes = model_to_dict(instance) object1 = camelcase_to_underscore(instance.__class__.__name__) @@ -476,7 +476,7 @@ def activity_stream_associate(sender, instance, **kwargs): return obj1 = instance _type = type(instance) - if hasattr(_type, '_deferred', False): + if hasattr(_type, '_deferred'): return object1=camelcase_to_underscore(obj1.__class__.__name__) obj_rel = sender.__module__ + "." + sender.__name__ @@ -489,7 +489,7 @@ def activity_stream_associate(sender, instance, **kwargs): continue obj2_actual = obj2_actual[0] _type = type(obj2_actual) - if hasattr(_type, '_deferred', False): + if hasattr(_type, '_deferred'): return if isinstance(obj2_actual, Role) and obj2_actual.content_object is not None: obj2_actual = obj2_actual.content_object From 6a4b4edea352816d2044a84cf1f8c5150031dfc2 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Sat, 16 Sep 2017 12:54:36 -0400 Subject: [PATCH 51/52] properly detect deferred ORM objects --- awx/main/signals.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/awx/main/signals.py b/awx/main/signals.py index 998f28cc11..77fd91a5c3 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -389,7 +389,7 @@ def activity_stream_create(sender, instance, created, **kwargs): if isinstance(instance, InventorySource) and instance.deprecated_group: return _type = type(instance) - if hasattr(_type, '_deferred'): + if getattr(_type, '_deferred', False): return object1 = camelcase_to_underscore(instance.__class__.__name__) changes = model_to_dict(instance, model_serializer_mapping) @@ -425,7 +425,7 @@ def activity_stream_update(sender, instance, **kwargs): if changes is None: return _type = type(instance) - if hasattr(_type, '_deferred'): + if getattr(_type, '_deferred', False): return object1 = camelcase_to_underscore(instance.__class__.__name__) activity_entry = ActivityStream( @@ -452,7 +452,7 @@ def activity_stream_delete(sender, instance, **kwargs): if isinstance(instance, Inventory) and not kwargs.get('inventory_delete_flag', False): return _type = type(instance) - if hasattr(_type, '_deferred'): + if getattr(_type, '_deferred', False): return changes = model_to_dict(instance) object1 = camelcase_to_underscore(instance.__class__.__name__) @@ -476,7 +476,7 @@ def activity_stream_associate(sender, instance, **kwargs): return obj1 = instance _type = type(instance) - if hasattr(_type, '_deferred'): + if getattr(_type, '_deferred', False): return object1=camelcase_to_underscore(obj1.__class__.__name__) obj_rel = sender.__module__ + "." + sender.__name__ @@ -489,7 +489,7 @@ def activity_stream_associate(sender, instance, **kwargs): continue obj2_actual = obj2_actual[0] _type = type(obj2_actual) - if hasattr(_type, '_deferred'): + if getattr(_type, '_deferred', False): return if isinstance(obj2_actual, Role) and obj2_actual.content_object is not None: obj2_actual = obj2_actual.content_object From 68b924efe5417f67d498a0522e5d320ff0feebe1 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 18 Sep 2017 09:44:39 -0400 Subject: [PATCH 52/52] flake8 fixup --- awx/main/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 2443e16f45..66aab8242f 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -146,6 +146,7 @@ activity_stream_registrar.connect(WorkflowJob) # prevent API filtering on certain Django-supplied sensitive fields prevent_search(User._meta.get_field('password')) + # Always, always, always defer result_stdout_text for polymorphic UnifiedJob rows # TODO: remove this defer in 3.3 when we implement https://github.com/ansible/ansible-tower/issues/5436 def defer_stdout(f):