From 16ebfe3a6373e7bd53e99d60b608a4f22880b355 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 9 Oct 2019 14:02:30 -0400 Subject: [PATCH 01/27] use fully qualified inventory plugin name --- Makefile | 2 +- awx_collection/plugins/inventory/tower.py | 4 ++-- awx_collection/template_galaxy.yml | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9fd0fd3db4..9b307613d4 100644 --- a/Makefile +++ b/Makefile @@ -400,7 +400,7 @@ flake8_collection: test_collection_all: prepare_collection_venv test_collection flake8_collection build_collection: - ansible-playbook -i localhost, awx_collection/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e namespace_name=$(COLLECTION_NAMESPACE) -e package_version=$(VERSION) + ansible-playbook -i localhost, awx_collection/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION) ansible-galaxy collection build awx_collection --output-path=awx_collection test_unit: diff --git a/awx_collection/plugins/inventory/tower.py b/awx_collection/plugins/inventory/tower.py index f4f20fe509..2cc35b339d 100644 --- a/awx_collection/plugins/inventory/tower.py +++ b/awx_collection/plugins/inventory/tower.py @@ -77,7 +77,7 @@ EXAMPLES = ''' # Example for using tower_inventory.yml file -plugin: tower +plugin: awx.awx.tower host: your_ansible_tower_server_network_address username: your_ansible_tower_username password: your_ansible_tower_password @@ -116,7 +116,7 @@ except ImportError: class InventoryModule(BaseInventoryPlugin): - NAME = 'tower' + NAME = 'awx.awx.tower' # REPLACE # Stays backward compatible with tower inventory script. # If the user supplies '@tower_inventory' as path, the plugin will read from environment variables. no_config_file_supplied = False diff --git a/awx_collection/template_galaxy.yml b/awx_collection/template_galaxy.yml index bfbea58bf7..f37198c46e 100644 --- a/awx_collection/template_galaxy.yml +++ b/awx_collection/template_galaxy.yml @@ -21,6 +21,12 @@ regexp: '^extends_documentation_fragment: awx.awx.auth$' replace: 'extends_documentation_fragment: {{ collection_namespace }}.{{ collection_package }}.auth' with_items: "{{ module_files.files }}" + + - name: Change files to support desired namespace and package names + replace: + path: "{{ playbook_dir }}/plugins/inventory/tower.py" + regexp: "^ NAME = 'awx.awx.tower' # REPLACE$" + replace: " NAME = '{{ collection_namespace }}.{{ collection_package }}.tower' # REPLACE" when: - (collection_package != 'awx') or (collection_namespace != 'awx') From d9dbbe6748c7a0c95d3e979642d39902b8b12055 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 10 Oct 2019 10:44:02 -0400 Subject: [PATCH 02/27] add a note about settings.LOG_AGGREGATOR_AUDIT usage see: https://github.com/ansible/awx/pull/4872#issuecomment-540133448 --- awx/main/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/conf.py b/awx/main/conf.py index 09e94485db..d75e254d1e 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -755,7 +755,7 @@ register( allow_null=True, default=False, label=_('Enabled external log aggregation auditing'), - help_text=_('When enabled, all external logs emitted by Tower will also be written to /var/log/tower/external.log'), + help_text=_('When enabled, all external logs emitted by Tower will also be written to /var/log/tower/external.log. This is an experimental setting intended to be used for debugging external log aggregation issues (and may be subject to change in the future).'), # noqa category=_('Logging'), category_slug='logging', ) From 008fe42b4deb4bad93699142f21083e7b8ec0161 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 10 Oct 2019 10:49:15 -0400 Subject: [PATCH 03/27] update to latest vmware_inventory.py https://github.com/ansible/ansible/blob/06c7b87613cc24b100a10074746d39e934eccfa7/contrib/inventory/vmware_inventory.py --- awx/plugins/inventory/vmware_inventory.py | 40 +++++++++++++++++------ 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/awx/plugins/inventory/vmware_inventory.py b/awx/plugins/inventory/vmware_inventory.py index 310194e164..183b9a19b0 100755 --- a/awx/plugins/inventory/vmware_inventory.py +++ b/awx/plugins/inventory/vmware_inventory.py @@ -39,8 +39,9 @@ import uuid from time import time from jinja2 import Environment -from six import integer_types, PY3 -from six.moves import configparser + +from ansible.module_utils.six import integer_types, PY3 +from ansible.module_utils.six.moves import configparser try: import argparse @@ -152,7 +153,7 @@ class VMWareInventory(object): try: text = str(text) except UnicodeEncodeError: - text = text.encode('ascii', 'ignore') + text = text.encode('utf-8') print('%s %s' % (datetime.datetime.now(), text)) def show(self): @@ -186,14 +187,14 @@ class VMWareInventory(object): def write_to_cache(self, data): ''' Dump inventory to json file ''' - with open(self.cache_path_cache, 'wb') as f: - f.write(json.dumps(data)) + with open(self.cache_path_cache, 'w') as f: + f.write(json.dumps(data, indent=2)) def get_inventory_from_cache(self): ''' Read in jsonified inventory ''' jdata = None - with open(self.cache_path_cache, 'rb') as f: + with open(self.cache_path_cache, 'r') as f: jdata = f.read() return json.loads(jdata) @@ -343,10 +344,22 @@ class VMWareInventory(object): 'pwd': self.password, 'port': int(self.port)} - if hasattr(ssl, 'SSLContext') and not self.validate_certs: + if self.validate_certs and hasattr(ssl, 'SSLContext'): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + kwargs['sslContext'] = context + elif self.validate_certs and not hasattr(ssl, 'SSLContext'): + sys.exit('pyVim does not support changing verification mode with python < 2.7.9. Either update ' + 'python or use validate_certs=false.') + elif not self.validate_certs and hasattr(ssl, 'SSLContext'): context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_NONE + context.check_hostname = False kwargs['sslContext'] = context + elif not self.validate_certs and not hasattr(ssl, 'SSLContext'): + # Python 2.7.9 < or RHEL/CentOS 7.4 < + pass return self._get_instances(kwargs) @@ -390,7 +403,7 @@ class VMWareInventory(object): instances = [x for x in instances if x.name == self.args.host] instance_tuples = [] - for instance in sorted(instances): + for instance in instances: if self.guest_props: ifacts = self.facts_from_proplist(instance) else: @@ -614,7 +627,14 @@ class VMWareInventory(object): lastref = lastref[x] else: lastref[x] = val - + if self.args.debug: + self.debugl("For %s" % vm.name) + for key in list(rdata.keys()): + if isinstance(rdata[key], dict): + for ikey in list(rdata[key].keys()): + self.debugl("Property '%s.%s' has value '%s'" % (key, ikey, rdata[key][ikey])) + else: + self.debugl("Property '%s' has value '%s'" % (key, rdata[key])) return rdata def facts_from_vobj(self, vobj, level=0): @@ -685,7 +705,7 @@ class VMWareInventory(object): if vobj.isalnum(): rdata = vobj else: - rdata = vobj.decode('ascii', 'ignore') + rdata = vobj.encode('utf-8').decode('utf-8') elif issubclass(type(vobj), bool) or isinstance(vobj, bool): rdata = vobj elif issubclass(type(vobj), integer_types) or isinstance(vobj, integer_types): From 31bdde00c9a415e2083fef285f189b31caca32b4 Mon Sep 17 00:00:00 2001 From: Bill Nottingham Date: Thu, 10 Oct 2019 15:08:20 -0400 Subject: [PATCH 04/27] Check the user's ansible.cfg for role/collection paths. There's no other way to add our new paths reliably without breaking things. --- awx/main/tasks.py | 15 ++++++++++++--- awx/main/utils/ansible.py | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index eb2d48546d..48d5a54086 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -73,6 +73,7 @@ from awx.main.utils import (get_ssh_version, update_scm_url, ignore_inventory_computed_fields, ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager, get_awx_version) +from awx.main.utils.ansible import read_ansible_config from awx.main.utils.common import get_ansible_version, _get_ansible_version, get_custom_venv_choices from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja from awx.main.utils.reload import stop_local_services @@ -1529,14 +1530,22 @@ class RunJob(BaseTask): if authorize: env['ANSIBLE_NET_AUTH_PASS'] = network_cred.get_input('authorize_password', default='') - for env_key, folder, default in ( - ('ANSIBLE_COLLECTIONS_PATHS', 'requirements_collections', '~/.ansible/collections:/usr/share/ansible/collections'), - ('ANSIBLE_ROLES_PATH', 'requirements_roles', '~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles')): + path_vars = ( + ('ANSIBLE_COLLECTIONS_PATHS', 'collections_paths', 'requirements_collections', '~/.ansible/collections:/usr/share/ansible/collections'), + ('ANSIBLE_ROLES_PATH', 'roles_path', 'requirements_roles', '~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles')) + + config_values = read_ansible_config(job.project.get_project_path(), list(map(lambda x: x[1], path_vars))) + + for env_key, config_setting, folder, default in path_vars: paths = default.split(':') if env_key in env: for path in env[env_key].split(':'): if path not in paths: paths = [env[env_key]] + paths + elif config_setting in config_values: + for path in config_values[config_setting].split(':'): + if path not in paths: + paths = [config_values[config_setting]] + paths paths = [os.path.join(private_data_dir, folder)] + paths env[env_key] = os.pathsep.join(paths) diff --git a/awx/main/utils/ansible.py b/awx/main/utils/ansible.py index 7e68d88189..287d5ba94f 100644 --- a/awx/main/utils/ansible.py +++ b/awx/main/utils/ansible.py @@ -5,11 +5,15 @@ import codecs import re import os +import logging from itertools import islice +from configparser import ConfigParser # Django from django.utils.encoding import smart_str +logger = logging.getLogger('awx.main.utils.ansible') + __all__ = ['skip_directory', 'could_be_playbook', 'could_be_inventory'] @@ -97,3 +101,20 @@ def could_be_inventory(project_path, dir_path, filename): except IOError: return None return inventory_rel_path + + +def read_ansible_config(project_path, variables_of_interest): + fnames = ['/etc/ansible/ansible.cfg'] + if project_path: + fnames.insert(0, os.path.join(project_path, 'ansible.cfg')) + values = {} + try: + parser = ConfigParser() + parser.read(fnames) + if 'defaults' in parser: + for var in variables_of_interest: + if var in parser['defaults']: + values[var] = parser['defaults'][var] + except Exception as e: + logger.warn('Failed to read ansible configuration(s) {}: {}'.format(fnames, e)) + return values From b93164e1ed3fe664941776147ff5166b53c23b7b Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 5 Oct 2019 14:07:33 -0400 Subject: [PATCH 05/27] Prevent pods from failing if the reason is because of a resource quota Signed-off-by: Shane McDonald --- awx/main/scheduler/kubernetes.py | 3 -- awx/main/tasks.py | 65 +++++++++++++++++++++++++------ awx/main/tests/unit/test_tasks.py | 3 ++ 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/awx/main/scheduler/kubernetes.py b/awx/main/scheduler/kubernetes.py index 1aa1978276..f4e35caec9 100644 --- a/awx/main/scheduler/kubernetes.py +++ b/awx/main/scheduler/kubernetes.py @@ -26,9 +26,6 @@ class PodManager(object): namespace=self.namespace, _request_timeout=settings.AWX_CONTAINER_GROUP_DEFAULT_LAUNCH_TIMEOUT) - # We don't do any fancy timeout logic here because it is handled - # at a higher level in the job spawning process. See - # settings.AWX_ISOLATED_LAUNCH_TIMEOUT and settings.AWX_ISOLATED_CONNECTION_TIMEOUT while True: pod = self.kube_api.read_namespaced_pod(name=self.pod_name, namespace=self.namespace, diff --git a/awx/main/tasks.py b/awx/main/tasks.py index eb2d48546d..6253446cd5 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -40,6 +40,9 @@ from django.utils.translation import ugettext_lazy as _ from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist +# Kubernetes +from kubernetes.client.rest import ApiException + # Django-CRUM from crum import impersonate @@ -1183,6 +1186,18 @@ class BaseTask(object): ''' Run the job/task and capture its output. ''' + self.instance = self.model.objects.get(pk=pk) + containerized = self.instance.is_containerized + pod_manager = None + if containerized: + # Here we are trying to launch a pod before transitioning the job into a running + # state. For some scenarios (like waiting for resources to become available) we do this + # rather than marking the job as error or failed. This is not always desirable. Cases + # such as invalid authentication should surface as an error. + pod_manager = self.deploy_container_group_pod(self.instance) + if not pod_manager: + return + # self.instance because of the update_model pattern and when it's used in callback handlers self.instance = self.update_model(pk, status='running', start_args='') # blank field to remove encrypted passwords @@ -1208,7 +1223,6 @@ class BaseTask(object): try: isolated = self.instance.is_isolated() - containerized = self.instance.is_containerized self.instance.send_notification_templates("running") private_data_dir = self.build_private_data_dir(self.instance) self.pre_run_hook(self.instance, private_data_dir) @@ -1287,6 +1301,10 @@ class BaseTask(object): }, } + if containerized: + # We don't want HOME passed through to container groups. + params['envvars'].pop('HOME') + if isinstance(self.instance, AdHocCommand): params['module'] = self.build_module_name(self.instance) params['module_args'] = self.build_module_args(self.instance) @@ -1316,16 +1334,6 @@ class BaseTask(object): params.pop('inventory'), os.path.join(private_data_dir, 'inventory') ) - pod_manager = None - if containerized: - from awx.main.scheduler.kubernetes import PodManager # Avoid circular import - params['envvars'].pop('HOME') - pod_manager = PodManager(self.instance) - self.cleanup_paths.append(pod_manager.kube_config) - pod_manager.deploy() - self.instance.execution_node = pod_manager.pod_name - self.instance.save(update_fields=['execution_node']) - ansible_runner.utils.dump_artifacts(params) isolated_manager_instance = isolated_manager.IsolatedManager( @@ -1385,6 +1393,41 @@ class BaseTask(object): raise AwxTaskError.TaskError(self.instance, rc) + def deploy_container_group_pod(self, task): + from awx.main.scheduler.kubernetes import PodManager # Avoid circular import + pod_manager = PodManager(self.instance) + self.cleanup_paths.append(pod_manager.kube_config) + try: + log_name = task.log_format + logger.debug(f"Launching pod for {log_name}.") + pod_manager.deploy() + except (ApiException, Exception) as exc: + if isinstance(exc, ApiException) and exc.status == 403: + try: + if 'exceeded quota' in json.loads(exc.body)['message']: + # If the k8s cluster does not have capacity, we move the job back into + # pending and immediately reschedule the task manager. + logger.warn(exc.body) + logger.warn(f"Could not launch pod for {log_name}. Exceeded quota.") + time.sleep(10) + self.update_model(task.pk, status='pending') + schedule_task_manager() + return + except Exception: + logger.exception(f"Unable to handle response from Kubernetes API for {log_name}.") + + logger.exception(f"Error when launching pod for {log_name}") + self.update_model(task.pk, status='error', result_traceback=exc.body) + return + + logger.debug(f"Pod online. Starting {log_name}.") + self.update_model(task.pk, execution_node=pod_manager.pod_name) + return pod_manager + + + + + @task() class RunJob(BaseTask): ''' diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 77c13fffd2..2bae5aef9a 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -368,6 +368,7 @@ class TestGenericRun(): task = tasks.RunJob() task.update_model = mock.Mock(return_value=job) + task.model.objects.get = mock.Mock(return_value=job) task.build_private_data_files = mock.Mock(side_effect=OSError()) with mock.patch('awx.main.tasks.copy_tree'): @@ -387,6 +388,7 @@ class TestGenericRun(): task = tasks.RunJob() task.update_model = mock.Mock(wraps=update_model_wrapper) + task.model.objects.get = mock.Mock(return_value=job) task.build_private_data_files = mock.Mock() with mock.patch('awx.main.tasks.copy_tree'): @@ -578,6 +580,7 @@ class TestAdhocRun(TestJobExecution): task = tasks.RunAdHocCommand() task.update_model = mock.Mock(wraps=adhoc_update_model_wrapper) + task.model.objects.get = mock.Mock(return_value=adhoc_job) task.build_inventory = mock.Mock() with pytest.raises(Exception): From 8f75382b81fa19bbdf966428eb4f80bdb22dfde2 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 5 Oct 2019 23:09:53 -0400 Subject: [PATCH 06/27] Implement retry logic for container group pod launches --- awx/main/scheduler/kubernetes.py | 23 +++++++++++++++-------- awx/main/tasks.py | 16 ++++++++++------ awx/settings/defaults.py | 4 +++- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/awx/main/scheduler/kubernetes.py b/awx/main/scheduler/kubernetes.py index f4e35caec9..90f2849c3d 100644 --- a/awx/main/scheduler/kubernetes.py +++ b/awx/main/scheduler/kubernetes.py @@ -3,6 +3,7 @@ import stat import time import yaml import tempfile +import logging from base64 import b64encode from django.conf import settings @@ -11,6 +12,8 @@ from django.utils.functional import cached_property from awx.main.utils.common import parse_yaml_or_json +logger = logging.getLogger('awx.main.scheduler') + class PodManager(object): @@ -21,29 +24,33 @@ class PodManager(object): if not self.credential.kubernetes: raise RuntimeError('Pod deployment cannot occur without a Kubernetes credential') - self.kube_api.create_namespaced_pod(body=self.pod_definition, namespace=self.namespace, - _request_timeout=settings.AWX_CONTAINER_GROUP_DEFAULT_LAUNCH_TIMEOUT) + _request_timeout=settings.AWX_CONTAINER_GROUP_K8S_API_TIMEOUT) - while True: + num_retries = settings.AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES + for retry_attempt in range(num_retries - 1): + logger.debug(f"Checking for pod {self.pod_name}. Attempt {retry_attempt + 1} of {num_retries}") pod = self.kube_api.read_namespaced_pod(name=self.pod_name, namespace=self.namespace, - _request_timeout=settings.AWX_CONTAINER_GROUP_DEFAULT_LAUNCH_TIMEOUT) + _request_timeout=settings.AWX_CONTAINER_GROUP_K8S_API_TIMEOUT) if pod.status.phase != 'Pending': break - time.sleep(1) + else: + logger.debug(f"Pod {self.pod_name} is Pending.") + time.sleep(settings.AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY) + continue if pod.status.phase == 'Running': + logger.debug(f"Pod {self.pod_name} is online.") return pod else: - raise RuntimeError(f"Unhandled Pod phase: {pod.status.phase}") - + logger.warn(f"Pod {self.pod_name} did not start. Status is {pod.status.phase}.") def delete(self): return self.kube_api.delete_namespaced_pod(name=self.pod_name, namespace=self.namespace, - _request_timeout=settings.AWX_CONTAINER_GROUP_DEFAULT_LAUNCH_TIMEOUT) + _request_timeout=settings.AWX_CONTAINER_GROUP_K8S_API_TIMEOUT) @property def namespace(self): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 6253446cd5..d46965ff46 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1405,13 +1405,15 @@ class BaseTask(object): if isinstance(exc, ApiException) and exc.status == 403: try: if 'exceeded quota' in json.loads(exc.body)['message']: - # If the k8s cluster does not have capacity, we move the job back into - # pending and immediately reschedule the task manager. + # If the k8s cluster does not have capacity, we move the + # job back into pending and wait until the next run of + # the task manager. This does not exactly play well with + # our current instance group precendence logic, since it + # will just sit here forever if kubernetes returns this + # error. logger.warn(exc.body) logger.warn(f"Could not launch pod for {log_name}. Exceeded quota.") - time.sleep(10) self.update_model(task.pk, status='pending') - schedule_task_manager() return except Exception: logger.exception(f"Unable to handle response from Kubernetes API for {log_name}.") @@ -1420,7 +1422,6 @@ class BaseTask(object): self.update_model(task.pk, status='error', result_traceback=exc.body) return - logger.debug(f"Pod online. Starting {log_name}.") self.update_model(task.pk, execution_node=pod_manager.pod_name) return pod_manager @@ -1833,7 +1834,10 @@ class RunJob(BaseTask): if job.is_containerized: from awx.main.scheduler.kubernetes import PodManager # prevent circular import - PodManager(job).delete() + pm = PodManager(job) + logger.debug(f"Deleting pod {pm.pod_name}") + pm.delete() + try: inventory = job.inventory diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 639d19f9e8..07c76f8b01 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -67,7 +67,9 @@ DATABASES = { } } -AWX_CONTAINER_GROUP_DEFAULT_LAUNCH_TIMEOUT = 10 +AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 +AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 +AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY = 5 AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE = 'default' AWX_CONTAINER_GROUP_DEFAULT_IMAGE = 'ansible/ansible-runner' From d5bdf554f1e15a9c6ad370137cbb43e478268565 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 9 Oct 2019 11:18:58 -0400 Subject: [PATCH 07/27] fix a programming error when k8s pods fail to launch --- awx/main/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index d46965ff46..66394a72d5 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1419,7 +1419,7 @@ class BaseTask(object): logger.exception(f"Unable to handle response from Kubernetes API for {log_name}.") logger.exception(f"Error when launching pod for {log_name}") - self.update_model(task.pk, status='error', result_traceback=exc.body) + self.update_model(task.pk, status='error', result_traceback=traceback.format_exc()) return self.update_model(task.pk, execution_node=pod_manager.pod_name) From a803cedd7ce3d5534ae9be524e7b098678f17dab Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Oct 2019 16:07:08 -0400 Subject: [PATCH 08/27] Break out a new reusable truncate_stdout utility function --- awx/api/serializers.py | 42 ++++++++++++-------------------------- awx/main/utils/common.py | 44 +++++++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 14d8944dc1..a5bdad1c39 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -45,7 +45,6 @@ from polymorphic.models import PolymorphicModel from awx.main.access import get_user_capabilities from awx.main.constants import ( SCHEDULEABLE_PROVIDERS, - ANSI_SGR_PATTERN, ACTIVE_STATES, CENSOR_VALUE, ) @@ -70,7 +69,8 @@ from awx.main.utils import ( get_type_for_model, get_model_for_type, camelcase_to_underscore, getattrd, parse_yaml_or_json, has_model_field_prefetched, extract_ansible_vars, encrypt_dict, - prefetch_page_capabilities, get_external_account) + prefetch_page_capabilities, get_external_account, truncate_stdout, +) from awx.main.utils.filters import SmartFilter from awx.main.redact import UriCleaner, REPLACE_STR @@ -3851,25 +3851,17 @@ class JobEventSerializer(BaseSerializer): return d def to_representation(self, obj): - ret = super(JobEventSerializer, self).to_representation(obj) + data = super(JobEventSerializer, self).to_representation(obj) # Show full stdout for event detail view, truncate only for list view. if hasattr(self.context.get('view', None), 'retrieve'): - return ret + return data # Show full stdout for playbook_on_* events. if obj and obj.event.startswith('playbook_on'): - return ret + return data max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY - if max_bytes > 0 and 'stdout' in ret and len(ret['stdout']) >= max_bytes: - ret['stdout'] = ret['stdout'][:(max_bytes - 1)] + u'\u2026' - set_count = 0 - reset_count = 0 - for m in ANSI_SGR_PATTERN.finditer(ret['stdout']): - if m.string[m.start():m.end()] == u'\u001b[0m': - reset_count += 1 - else: - set_count += 1 - ret['stdout'] += u'\u001b[0m' * (set_count - reset_count) - return ret + if 'stdout' in data: + data['stdout'] = truncate_stdout(data['stdout'], max_bytes) + return data class JobEventWebSocketSerializer(JobEventSerializer): @@ -3964,22 +3956,14 @@ class AdHocCommandEventSerializer(BaseSerializer): return res def to_representation(self, obj): - ret = super(AdHocCommandEventSerializer, self).to_representation(obj) + data = super(AdHocCommandEventSerializer, self).to_representation(obj) # Show full stdout for event detail view, truncate only for list view. if hasattr(self.context.get('view', None), 'retrieve'): - return ret + return data max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY - if max_bytes > 0 and 'stdout' in ret and len(ret['stdout']) >= max_bytes: - ret['stdout'] = ret['stdout'][:(max_bytes - 1)] + u'\u2026' - set_count = 0 - reset_count = 0 - for m in ANSI_SGR_PATTERN.finditer(ret['stdout']): - if m.string[m.start():m.end()] == u'\u001b[0m': - reset_count += 1 - else: - set_count += 1 - ret['stdout'] += u'\u001b[0m' * (set_count - reset_count) - return ret + if 'stdout' in data: + data['stdout'] = truncate_stdout(data['stdout'], max_bytes) + return data class AdHocCommandEventWebSocketSerializer(AdHocCommandEventSerializer): diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index d36dfa272b..11e4722f5f 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -38,18 +38,22 @@ from django.apps import apps logger = logging.getLogger('awx.main.utils') -__all__ = ['get_object_or_400', 'camelcase_to_underscore', 'underscore_to_camelcase', 'memoize', 'memoize_delete', - 'get_ansible_version', 'get_ssh_version', 'get_licenser', 'get_awx_version', 'update_scm_url', - 'get_type_for_model', 'get_model_for_type', 'copy_model_by_class', 'region_sorting', - 'copy_m2m_relationships', 'prefetch_page_capabilities', 'to_python_boolean', - 'ignore_inventory_computed_fields', 'ignore_inventory_group_removal', - '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'getattr_dne', 'NoDefaultProvided', - 'get_current_apps', 'set_current_apps', - 'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity', 'get_cpu_capacity', 'get_mem_capacity', - 'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict', - 'NullablePromptPseudoField', 'model_instance_diff', 'parse_yaml_or_json', 'RequireDebugTrueOrTest', - 'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', 'get_custom_venv_choices', 'get_external_account', - 'task_manager_bulk_reschedule', 'schedule_task_manager', 'classproperty', 'create_temporary_fifo'] +__all__ = [ + 'get_object_or_400', 'camelcase_to_underscore', 'underscore_to_camelcase', 'memoize', + 'memoize_delete', 'get_ansible_version', 'get_ssh_version', 'get_licenser', + 'get_awx_version', 'update_scm_url', 'get_type_for_model', 'get_model_for_type', + 'copy_model_by_class', 'region_sorting', 'copy_m2m_relationships', + 'prefetch_page_capabilities', 'to_python_boolean', 'ignore_inventory_computed_fields', + 'ignore_inventory_group_removal', '_inventory_updates', 'get_pk_from_dict', 'getattrd', + 'getattr_dne', 'NoDefaultProvided', 'get_current_apps', 'set_current_apps', + 'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity', + 'get_cpu_capacity', 'get_mem_capacity', 'wrap_args_with_proot', 'build_proot_temp_dir', + 'check_proot_installed', 'model_to_dict', 'NullablePromptPseudoField', + 'model_instance_diff', 'parse_yaml_or_json', 'RequireDebugTrueOrTest', + 'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', + 'get_custom_venv_choices', 'get_external_account', 'task_manager_bulk_reschedule', + 'schedule_task_manager', 'classproperty', 'create_temporary_fifo', 'truncate_stdout', +] def get_object_or_400(klass, *args, **kwargs): @@ -1088,3 +1092,19 @@ def create_temporary_fifo(data): ).start() return path + +def truncate_stdout(stdout, size): + from awx.main.constants import ANSI_SGR_PATTERN + + if size <= 0 or len(stdout) <= size: + return stdout + + stdout = stdout[:(size - 1)] + u'\u2026' + set_count, reset_count = 0, 0 + for m in ANSI_SGR_PATTERN.finditer(stdout): + if m.group() == u'\u001b[0m': + reset_count += 1 + else: + set_count += 1 + + return stdout + u'\u001b[0m' * (set_count - reset_count) From 9efa7b84dfbb0e4654f440f6b8cdc48b79789343 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Oct 2019 16:08:17 -0400 Subject: [PATCH 09/27] Depend on a serializer context variable `no_truncate` to decide whether to turn off the ANSI control sequence-aware truncation, instead of needing inappropriate awareness of the details of the view that invoked the serializer. This will also allow us to have views that can more flexibly turn off the truncation under other circumstances. --- awx/api/serializers.py | 10 +++++----- awx/api/views/__init__.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index a5bdad1c39..71525ccf0c 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -3852,12 +3852,12 @@ class JobEventSerializer(BaseSerializer): def to_representation(self, obj): data = super(JobEventSerializer, self).to_representation(obj) - # Show full stdout for event detail view, truncate only for list view. - if hasattr(self.context.get('view', None), 'retrieve'): - return data # Show full stdout for playbook_on_* events. if obj and obj.event.startswith('playbook_on'): return data + # If the view logic says to not trunctate (request was to the detail view or a param was used) + if self.context.get('no_truncate', False): + return data max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY if 'stdout' in data: data['stdout'] = truncate_stdout(data['stdout'], max_bytes) @@ -3957,8 +3957,8 @@ class AdHocCommandEventSerializer(BaseSerializer): def to_representation(self, obj): data = super(AdHocCommandEventSerializer, self).to_representation(obj) - # Show full stdout for event detail view, truncate only for list view. - if hasattr(self.context.get('view', None), 'retrieve'): + # If the view logic says to not trunctate (request was to the detail view or a param was used) + if self.context.get('no_truncate', False): return data max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY if 'stdout' in data: diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 383c7aeee9..646ff2c746 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3774,6 +3774,11 @@ class JobEventDetail(RetrieveAPIView): model = models.JobEvent serializer_class = serializers.JobEventSerializer + def get_serializer_context(self): + context = super().get_serializer_context() + context.update(no_truncate=True) + return context + class JobEventChildrenList(SubListAPIView): @@ -4008,6 +4013,11 @@ class AdHocCommandEventDetail(RetrieveAPIView): model = models.AdHocCommandEvent serializer_class = serializers.AdHocCommandEventSerializer + def get_serializer_context(self): + context = super().get_serializer_context() + context.update(no_truncate=True) + return context + class BaseAdHocCommandEventsList(SubListAPIView): From 08839e13812d49b6870e473ae323d5f6ad79f47f Mon Sep 17 00:00:00 2001 From: Graham Mainwaring Date: Fri, 11 Oct 2019 12:50:43 -0400 Subject: [PATCH 10/27] Add approved_by field to workflow approvals --- awx/api/serializers.py | 3 +++ ..._workflowapproval_approved_or_denied_by.py | 21 +++++++++++++++++++ awx/main/models/workflow.py | 13 ++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 awx/main/migrations/0097_v360_workflowapproval_approved_or_denied_by.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 14d8944dc1..3199cdec84 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -140,6 +140,7 @@ SUMMARIZABLE_FK_FIELDS = { 'source_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'), 'target_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'), 'webhook_credential': DEFAULT_SUMMARY_FIELDS, + 'approved_or_denied_by': ('id', 'username', 'first_name', 'last_name'), } @@ -3501,6 +3502,8 @@ class WorkflowApprovalSerializer(UnifiedJobSerializer): kwargs={'pk': obj.workflow_approval_template.pk}) res['approve'] = self.reverse('api:workflow_approval_approve', kwargs={'pk': obj.pk}) res['deny'] = self.reverse('api:workflow_approval_deny', kwargs={'pk': obj.pk}) + if obj.approved_or_denied_by: + res['approved_or_denied_by'] = self.reverse('api:user_detail', kwargs={'pk': obj.approved_or_denied_by.pk}) return res diff --git a/awx/main/migrations/0097_v360_workflowapproval_approved_or_denied_by.py b/awx/main/migrations/0097_v360_workflowapproval_approved_or_denied_by.py new file mode 100644 index 0000000000..84bf80c7f6 --- /dev/null +++ b/awx/main/migrations/0097_v360_workflowapproval_approved_or_denied_by.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.4 on 2019-10-11 15:40 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('main', '0096_v360_container_groups'), + ] + + operations = [ + migrations.AddField( + model_name='workflowapproval', + name='approved_or_denied_by', + field=models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'workflowapproval', 'model_name': 'workflowapproval', 'app_label': 'main'}(class)s_approved+", to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 56f5cfd579..bcfee585b8 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -13,6 +13,9 @@ from django.utils.translation import ugettext_lazy as _ from django.core.exceptions import ObjectDoesNotExist #from django import settings as tower_settings +# Django-CRUM +from crum import get_current_user + # AWX from awx.api.versioning import reverse from awx.main.models import (prevent_search, accepts_json, UnifiedJobTemplate, @@ -690,6 +693,14 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin): default=False, help_text=_("Shows when an approval node (with a timeout assigned to it) has timed out.") ) + approved_or_denied_by = models.ForeignKey( + 'auth.User', + related_name='%s(class)s_approved+', + default=None, + null=True, + editable=False, + on_delete=models.SET_NULL, + ) @classmethod @@ -711,6 +722,7 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin): def approve(self, request=None): self.status = 'successful' + self.approved_or_denied_by = get_current_user() self.save() self.send_approval_notification('approved') self.websocket_emit_status(self.status) @@ -719,6 +731,7 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin): def deny(self, request=None): self.status = 'failed' + self.approved_or_denied_by = get_current_user() self.save() self.send_approval_notification('denied') self.websocket_emit_status(self.status) From e672e68a02d03090db7868fd7ca60613950f9b7e Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Oct 2019 16:21:53 -0400 Subject: [PATCH 11/27] Allow the job event list views to take a no_truncate GET param --- awx/api/filters.py | 2 +- awx/api/views/__init__.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/awx/api/filters.py b/awx/api/filters.py index 327303dd2e..ea9d011562 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -126,7 +126,7 @@ class FieldLookupBackend(BaseFilterBackend): ''' RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', - 'search', 'type', 'host_filter', 'count_disabled',) + 'search', 'type', 'host_filter', 'count_disabled', 'no_truncate') SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 646ff2c746..f337345df9 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3768,6 +3768,12 @@ class JobEventList(ListAPIView): serializer_class = serializers.JobEventSerializer search_fields = ('stdout',) + def get_serializer_context(self): + context = super().get_serializer_context() + if self.request.query_params.get('no_truncate'): + context.update(no_truncate=True) + return context + class JobEventDetail(RetrieveAPIView): @@ -4007,6 +4013,12 @@ class AdHocCommandEventList(ListAPIView): serializer_class = serializers.AdHocCommandEventSerializer search_fields = ('stdout',) + def get_serializer_context(self): + context = super().get_serializer_context() + if self.request.query_params.get('no_truncate'): + context.update(no_truncate=True) + return context + class AdHocCommandEventDetail(RetrieveAPIView): From cf89108edffeea2b2e594150159193eb5eb0be88 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Oct 2019 16:57:39 -0400 Subject: [PATCH 12/27] Force the CLI to use no_truncate for the monitor calls --- awxkit/awxkit/cli/stdout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/awxkit/cli/stdout.py b/awxkit/awxkit/cli/stdout.py index 47ca7f79f6..1cf18168e1 100644 --- a/awxkit/awxkit/cli/stdout.py +++ b/awxkit/awxkit/cli/stdout.py @@ -73,7 +73,7 @@ def monitor_workflow(response, session, print_stdout=True, timeout=None, def monitor(response, session, print_stdout=True, timeout=None, interval=.25): get = response.url.get - payload = {'order_by': 'start_line'} + payload = {'order_by': 'start_line', 'no_truncate': True} if response.type == 'job': events = response.related.job_events.get else: From 8e26e4edd59b6bae119377f9b7a46dca972c07f6 Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Mon, 14 Oct 2019 11:38:20 -0400 Subject: [PATCH 13/27] Allow oauth2 settings to be set in the ui and api Oauth2 settings were initialized early in the awx import stage, and those settings were not modifiable. This change allows oauth2 to check for settings in django.conf settings, which are dynamically updated through api calls at runtime. As a result, oauth2 settings will match the values in django.conf settings at any point in time. --- awx/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/awx/__init__.py b/awx/__init__.py index e7a5a828ec..b97a5694af 100644 --- a/awx/__init__.py +++ b/awx/__init__.py @@ -82,6 +82,16 @@ def find_commands(management_dir): return commands +def oauth2_getattribute(self, attr): + # Custom method to override + # oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__ + from django.conf import settings + val = settings.OAUTH2_PROVIDER.get(attr) + if val is None: + val = object.__getattribute__(self, attr) + return val + + def prepare_env(): # Update the default settings environment variable based on current mode. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE) @@ -93,6 +103,12 @@ def prepare_env(): # Monkeypatch Django find_commands to also work with .pyc files. import django.core.management django.core.management.find_commands = find_commands + + # Monkeypatch Oauth2 toolkit settings class to check for settings + # in django.conf settings each time, not just once during import + import oauth2_provider.settings + oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__ = oauth2_getattribute + # Use the AWX_TEST_DATABASE_* environment variables to specify the test # database settings to use when management command is run as an external # program via unit tests. From 6282b5bacbb30f31bc7bbf53df668ac8ed68829f Mon Sep 17 00:00:00 2001 From: Marliana Lara Date: Mon, 14 Oct 2019 13:11:31 -0400 Subject: [PATCH 14/27] Style empty list placeholder text inline --- awx/ui/client/legacy/styles/lists.less | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/awx/ui/client/legacy/styles/lists.less b/awx/ui/client/legacy/styles/lists.less index 3e004461af..8357a78b78 100644 --- a/awx/ui/client/legacy/styles/lists.less +++ b/awx/ui/client/legacy/styles/lists.less @@ -372,9 +372,7 @@ table, tbody { .List-noItems { margin-top: 52px; - display: flex; - align-items: center; - justify-content: center; + display: inline-block; width: 100%; height: 200px; border-radius: 5px; @@ -383,7 +381,7 @@ table, tbody { color: @list-no-items-txt; text-transform: uppercase; text-align: center; - padding: 10px; + padding: 80px 10px; } .modal-body > .List-noItems { From e5184e0ed14f848d992809084b2d0ee9b7268276 Mon Sep 17 00:00:00 2001 From: Marliana Lara Date: Mon, 14 Oct 2019 13:50:59 -0400 Subject: [PATCH 15/27] Fix workflow results detail panel responsive style --- awx/ui/client/src/shared/layouts/one-plus-two.less | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/awx/ui/client/src/shared/layouts/one-plus-two.less b/awx/ui/client/src/shared/layouts/one-plus-two.less index 067e050e27..c9514daae5 100644 --- a/awx/ui/client/src/shared/layouts/one-plus-two.less +++ b/awx/ui/client/src/shared/layouts/one-plus-two.less @@ -20,10 +20,11 @@ flex: 1 0; height: @height; width: 100%; - margin-right: 20px; - @media screen and (max-width: @breakpoint){ - margin-right: 0px; - height: inherit; + margin-right: 20px; + @media screen and (max-width: @breakpoint){ + height: inherit; + margin-right: 0px; + max-width: none; } } From 85781d0bc152b6ac1b845e9fa89d8ab4eef5e4ee Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Mon, 14 Oct 2019 13:42:39 -0400 Subject: [PATCH 16/27] Allow navigation to previous launch prompt tabs --- .../lib/components/tabs/tab.directive.js | 12 ++-- .../lib/components/tabs/tab.partial.html | 2 +- .../src/templates/prompt/prompt.controller.js | 70 ++++++++++++++++--- .../src/templates/prompt/prompt.partial.html | 2 +- 4 files changed, 69 insertions(+), 17 deletions(-) diff --git a/awx/ui/client/lib/components/tabs/tab.directive.js b/awx/ui/client/lib/components/tabs/tab.directive.js index 4c7eea3785..85dc5f4d2c 100644 --- a/awx/ui/client/lib/components/tabs/tab.directive.js +++ b/awx/ui/client/lib/components/tabs/tab.directive.js @@ -20,16 +20,18 @@ function AtTabController ($state) { group.register(scope); }; - vm.go = () => { + vm.handleClick = () => { if (scope.state._disabled || scope.state._active) { return; } - if (scope.state._go) { $state.go(scope.state._go, scope.state._params, { reload: true }); - } else { - group.clearActive(); - scope.state._active = true; + return; + } + group.clearActive(); + scope.state._active = true; + if (scope.state._onClickActivate) { + scope.state._onClickActivate(); } }; } diff --git a/awx/ui/client/lib/components/tabs/tab.partial.html b/awx/ui/client/lib/components/tabs/tab.partial.html index eb007feb01..5b77e02841 100644 --- a/awx/ui/client/lib/components/tabs/tab.partial.html +++ b/awx/ui/client/lib/components/tabs/tab.partial.html @@ -2,6 +2,6 @@ ng-attr-disabled="{{ state._disabled || undefined }}" ng-class="{ 'at-Tab--active': state._active, 'at-Tab--disabled': state._disabled }" ng-hide="{{ state._hide }}" - ng-click="state._go && vm.go();"> + ng-click="vm.handleClick();"> diff --git a/awx/ui/client/src/templates/prompt/prompt.controller.js b/awx/ui/client/src/templates/prompt/prompt.controller.js index 5aeff6e63b..cf92af6258 100644 --- a/awx/ui/client/src/templates/prompt/prompt.controller.js +++ b/awx/ui/client/src/templates/prompt/prompt.controller.js @@ -189,6 +189,36 @@ export default [ 'ProcessErrors', 'CredentialTypeModel', 'TemplatesStrings', '$f modal.show($filter('sanitize')(vm.promptDataClone.templateName)); vm.promptData.triggerModalOpen = false; + vm._savedPromptData = { + 1: _.cloneDeep(vm.promptDataClone) + }; + Object.keys(vm.steps).forEach(step => { + if (!vm.steps[step].tab) { + return; + } + vm.steps[step].tab._onClickActivate = () => { + if (vm._savedPromptData[vm.steps[step].tab.order]) { + vm.promptDataClone = vm._savedPromptData[vm.steps[step].tab.order]; + } + Object.keys(vm.steps).forEach(tabStep => { + if (!vm.steps[tabStep].tab) { + return; + } + if (vm.steps[tabStep].tab.order < vm.steps[step].tab.order) { + vm.steps[tabStep].tab._disabled = false; + vm.steps[tabStep].tab._active = false; + } else if (vm.steps[tabStep].tab.order === vm.steps[step].tab.order) { + vm.steps[tabStep].tab._disabled = false; + vm.steps[tabStep].tab._active = true; + } else { + vm.steps[tabStep].tab._disabled = true; + vm.steps[tabStep].tab._active = false; + } + }); + scope.$broadcast('promptTabChange', { step }); + }; + }); + modal.onClose = () => { scope.$emit('launchModalOpen', false); }; @@ -214,19 +244,39 @@ export default [ 'ProcessErrors', 'CredentialTypeModel', 'TemplatesStrings', '$f return; } } + + let nextStep; Object.keys(vm.steps).forEach(step => { - if(vm.steps[step].tab) { - if(vm.steps[step].tab.order === currentTab.order) { - vm.steps[step].tab._active = false; - vm.steps[step].tab._disabled = true; - } else if(vm.steps[step].tab.order === currentTab.order + 1) { - activeTab = currentTab; - vm.steps[step].tab._active = true; - vm.steps[step].tab._disabled = false; - scope.$broadcast('promptTabChange', { step }); - } + if (!vm.steps[step].tab) { + return; + } + if (vm.steps[step].tab.order === currentTab.order + 1) { + nextStep = step; } }); + + if (!nextStep) { + return; + } + + // Save the current promptData state in case we need to revert + vm._savedPromptData[currentTab.order] = _.cloneDeep(vm.promptDataClone); + Object.keys(vm.steps).forEach(tabStep => { + if (!vm.steps[tabStep].tab) { + return; + } + if (vm.steps[tabStep].tab.order < vm.steps[nextStep].tab.order) { + vm.steps[tabStep].tab._disabled = false; + vm.steps[tabStep].tab._active = false; + } else if (vm.steps[tabStep].tab.order === vm.steps[nextStep].tab.order) { + vm.steps[tabStep].tab._disabled = false; + vm.steps[tabStep].tab._active = true; + } else { + vm.steps[tabStep].tab._disabled = true; + vm.steps[tabStep].tab._active = false; + } + }); + scope.$broadcast('promptTabChange', { step: nextStep }); }; vm.keypress = (event) => { diff --git a/awx/ui/client/src/templates/prompt/prompt.partial.html b/awx/ui/client/src/templates/prompt/prompt.partial.html index 6bd2a123af..217ddddf94 100644 --- a/awx/ui/client/src/templates/prompt/prompt.partial.html +++ b/awx/ui/client/src/templates/prompt/prompt.partial.html @@ -22,7 +22,7 @@ read-only-prompts="vm.readOnlyPrompts"> -
+
Date: Fri, 11 Oct 2019 12:03:17 -0400 Subject: [PATCH 17/27] Sys Aud can see CG forms, Adds correct CG form link, Disables CodeMirror This allows the System Auditor to see the container groups form in a disabled state. If the pod_spec_override has been changed that field will be open when the page renders but it will be disabled. It also greys out all code mirror text area fields for System Auditor. It adds the correct url for the Container Groups message bar to inform users of possible pitfalls associated with that feature. --- .../lib/components/code-mirror/_index.less | 5 ++++ .../add-container-group.view.html | 10 ++++--- .../edit-container-group.controller.js | 29 +++++++++++++------ .../instance-groups/instance-group.block.less | 1 + 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/awx/ui/client/lib/components/code-mirror/_index.less b/awx/ui/client/lib/components/code-mirror/_index.less index 6eeb936e79..d0e2d970e8 100644 --- a/awx/ui/client/lib/components/code-mirror/_index.less +++ b/awx/ui/client/lib/components/code-mirror/_index.less @@ -52,6 +52,11 @@ height: calc(~"100vh - 80px"); } +textarea[disabled] + div .CodeMirror-code{ + background-color: #ebebeb; + cursor: not-allowed; +} + @media screen and (min-width: 768px){ .NetworkingExtraVars .modal-dialog{ diff --git a/awx/ui/client/src/instance-groups/container-groups/add-container-group.view.html b/awx/ui/client/src/instance-groups/container-groups/add-container-group.view.html index 86ae509309..df8e6afee7 100644 --- a/awx/ui/client/src/instance-groups/container-groups/add-container-group.view.html +++ b/awx/ui/client/src/instance-groups/container-groups/add-container-group.view.html @@ -1,5 +1,5 @@
- +
This feature is tech preview, and is subject to change in a future release. Click here for documentation. @@ -21,13 +21,15 @@ {{ vm.form.extraVars.toggleLabel }} -
- +
+
Date: Tue, 15 Oct 2019 14:20:09 -0400 Subject: [PATCH 18/27] Use logger.exception instead of logger.warning. --- awx/main/utils/ansible.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/utils/ansible.py b/awx/main/utils/ansible.py index 287d5ba94f..06f56383c1 100644 --- a/awx/main/utils/ansible.py +++ b/awx/main/utils/ansible.py @@ -116,5 +116,5 @@ def read_ansible_config(project_path, variables_of_interest): if var in parser['defaults']: values[var] = parser['defaults'][var] except Exception as e: - logger.warn('Failed to read ansible configuration(s) {}: {}'.format(fnames, e)) + logger.exception('Failed to read ansible configuration(s) {}'.format(fnames)) return values From 2123092bdcbbebbf775549e4a92d18d7e0a1560e Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 16 Oct 2019 09:08:22 -0400 Subject: [PATCH 19/27] Avoid unnecessary OPTIONS redirect --- awxkit/awxkit/cli/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/awxkit/cli/options.py b/awxkit/awxkit/cli/options.py index 858808ae8e..5156482df9 100644 --- a/awxkit/awxkit/cli/options.py +++ b/awxkit/awxkit/cli/options.py @@ -95,7 +95,7 @@ class ResourceOptionsParser(object): def get_allowed_options(self): self.allowed_options = self.page.connection.options( - self.page.endpoint + '1' + self.page.endpoint + '1/' ).headers.get('Allow', '').split(', ') def build_list_actions(self): From 4134d0b51640c2913fd65895a8f0ed409f92f944 Mon Sep 17 00:00:00 2001 From: Alex Corey Date: Wed, 16 Oct 2019 08:36:22 -0400 Subject: [PATCH 20/27] Updates PR and Addresses Console Error THis commit imroves conditional rendering of the container groups form for System Auditors. It also removes a ng-class condition in the IG list that was unused. --- awx/ui/client/lib/components/code-mirror/_index.less | 6 ------ .../container-groups/edit-container-group.controller.js | 5 ++++- .../instance-groups/list/instance-groups-list.partial.html | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/awx/ui/client/lib/components/code-mirror/_index.less b/awx/ui/client/lib/components/code-mirror/_index.less index d0e2d970e8..98c258d528 100644 --- a/awx/ui/client/lib/components/code-mirror/_index.less +++ b/awx/ui/client/lib/components/code-mirror/_index.less @@ -52,12 +52,6 @@ height: calc(~"100vh - 80px"); } -textarea[disabled] + div .CodeMirror-code{ - background-color: #ebebeb; - cursor: not-allowed; -} - - @media screen and (min-width: 768px){ .NetworkingExtraVars .modal-dialog{ width: 700px; diff --git a/awx/ui/client/src/instance-groups/container-groups/edit-container-group.controller.js b/awx/ui/client/src/instance-groups/container-groups/edit-container-group.controller.js index 76559e37fd..33d50cb758 100644 --- a/awx/ui/client/src/instance-groups/container-groups/edit-container-group.controller.js +++ b/awx/ui/client/src/instance-groups/container-groups/edit-container-group.controller.js @@ -4,7 +4,10 @@ function EditContainerGroupController($rootScope, $scope, $state, models, string instanceGroup, credential } = models; - const canEdit = instanceGroup.model.OPTIONS.actions.PUT; + let canEdit = false; + if (instanceGroup.has('options', 'actions.PUT')) { + canEdit = instanceGroup.model.OPTIONS.actions.PUT; +} if (!instanceGroup.get('is_containerized')) { return $state.go( 'instanceGroups.edit', diff --git a/awx/ui/client/src/instance-groups/list/instance-groups-list.partial.html b/awx/ui/client/src/instance-groups/list/instance-groups-list.partial.html index 88537cee03..b8d5fdfbc3 100644 --- a/awx/ui/client/src/instance-groups/list/instance-groups-list.partial.html +++ b/awx/ui/client/src/instance-groups/list/instance-groups-list.partial.html @@ -101,7 +101,7 @@
- +
From cd18ec408c71b178d5fc50ffa990856cf72ce552 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Wed, 16 Oct 2019 09:30:43 -0400 Subject: [PATCH 21/27] Remove unused variable --- awx/main/utils/ansible.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/utils/ansible.py b/awx/main/utils/ansible.py index 06f56383c1..fa15c47ab1 100644 --- a/awx/main/utils/ansible.py +++ b/awx/main/utils/ansible.py @@ -115,6 +115,6 @@ def read_ansible_config(project_path, variables_of_interest): for var in variables_of_interest: if var in parser['defaults']: values[var] = parser['defaults'][var] - except Exception as e: + except Exception: logger.exception('Failed to read ansible configuration(s) {}'.format(fnames)) return values From 86ef81cebf5c5a7cf841194992e2f873089a0344 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 16 Oct 2019 09:34:21 -0400 Subject: [PATCH 22/27] API deprecation of inventory script views --- awx/api/views/inventory.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/awx/api/views/inventory.py b/awx/api/views/inventory.py index dfa3f6627a..987f5467b4 100644 --- a/awx/api/views/inventory.py +++ b/awx/api/views/inventory.py @@ -70,12 +70,16 @@ class InventoryUpdateEventsList(SubListAPIView): class InventoryScriptList(ListCreateAPIView): + deprecated = True + model = CustomInventoryScript serializer_class = CustomInventoryScriptSerializer class InventoryScriptDetail(RetrieveUpdateDestroyAPIView): + deprecated = True + model = CustomInventoryScript serializer_class = CustomInventoryScriptSerializer @@ -92,6 +96,8 @@ class InventoryScriptDetail(RetrieveUpdateDestroyAPIView): class InventoryScriptObjectRolesList(SubListAPIView): + deprecated = True + model = Role serializer_class = RoleSerializer parent_model = CustomInventoryScript @@ -105,6 +111,8 @@ class InventoryScriptObjectRolesList(SubListAPIView): class InventoryScriptCopy(CopyAPIView): + deprecated = True + model = CustomInventoryScript copy_return_serializer_class = CustomInventoryScriptSerializer From 5f2e1c9705cd04f6d09d8d3ea1480573449c0908 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 16 Oct 2019 09:42:39 -0400 Subject: [PATCH 23/27] fix a tz parsing bug --- awx/main/models/schedules.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/awx/main/models/schedules.py b/awx/main/models/schedules.py index d4a5f12914..58aee91d96 100644 --- a/awx/main/models/schedules.py +++ b/awx/main/models/schedules.py @@ -119,10 +119,11 @@ class Schedule(PrimordialModel, LaunchTimeConfig): tzinfo = r._dtstart.tzinfo if tzinfo is utc: return 'UTC' - fname = tzinfo._filename - for zone in all_zones: - if fname.endswith(zone): - return zone + fname = getattr(tzinfo, '_filename', None) + if fname: + for zone in all_zones: + if fname.endswith(zone): + return zone logger.warn('Could not detect valid zoneinfo for {}'.format(self.rrule)) return '' From ad89c5eea732d6bf18a52062b035fafaa077b620 Mon Sep 17 00:00:00 2001 From: beeankha Date: Thu, 10 Oct 2019 16:38:15 -0400 Subject: [PATCH 24/27] Enable approval notification support for CLI --- awxkit/awxkit/cli/custom.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/awxkit/awxkit/cli/custom.py b/awxkit/awxkit/cli/custom.py index ad2af3b433..903f2e4fc7 100644 --- a/awxkit/awxkit/cli/custom.py +++ b/awxkit/awxkit/cli/custom.py @@ -225,6 +225,7 @@ class AssociationMixin(object): def __init__(self, connection, resource): self.conn = connection self.resource = { + 'approval_notification': 'notification_templates', 'start_notification': 'notification_templates', 'success_notification': 'notification_templates', 'failure_notification': 'notification_templates', @@ -299,11 +300,27 @@ JobTemplateNotificationDisAssociation.targets.update({ class WorkflowJobTemplateNotificationAssociation(NotificationAssociateMixin, CustomAction): resource = 'workflow_job_templates' action = 'associate' + targets = dict( + **NotificationAssociateMixin.targets, + **{'approval_notification': [ + 'notification_templates_approvals', + 'notification_template' + ] + }, + ) class WorkflowJobTemplateNotificationDisAssociation(NotificationAssociateMixin, CustomAction): resource = 'workflow_job_templates' action = 'disassociate' + targets = dict( + **NotificationAssociateMixin.targets, + **{'approval_notification': [ + 'notification_templates_approvals', + 'notification_template' + ] + }, + ) class ProjectNotificationAssociation(NotificationAssociateMixin, CustomAction): @@ -329,11 +346,27 @@ class InventorySourceNotificationDisAssociation(NotificationAssociateMixin, Cust class OrganizationNotificationAssociation(NotificationAssociateMixin, CustomAction): resource = 'organizations' action = 'associate' + targets = dict( + **NotificationAssociateMixin.targets, + **{'approval_notification': [ + 'notification_templates_approvals', + 'notification_template' + ] + }, + ) class OrganizationNotificationDisAssociation(NotificationAssociateMixin, CustomAction): resource = 'organizations' action = 'disassociate' + targets = dict( + **NotificationAssociateMixin.targets, + **{'approval_notification': [ + 'notification_templates_approvals', + 'notification_template' + ] + }, + ) class SettingsList(CustomAction): From fdddba18be16a6bacc4ce4a28d1c8d324949ae49 Mon Sep 17 00:00:00 2001 From: beeankha Date: Fri, 11 Oct 2019 10:05:03 -0400 Subject: [PATCH 25/27] Update code to be compatible with py2 --- awxkit/awxkit/cli/custom.py | 51 ++++++++++++++----------------------- 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/awxkit/awxkit/cli/custom.py b/awxkit/awxkit/cli/custom.py index 903f2e4fc7..f9077acab5 100644 --- a/awxkit/awxkit/cli/custom.py +++ b/awxkit/awxkit/cli/custom.py @@ -300,27 +300,21 @@ JobTemplateNotificationDisAssociation.targets.update({ class WorkflowJobTemplateNotificationAssociation(NotificationAssociateMixin, CustomAction): resource = 'workflow_job_templates' action = 'associate' - targets = dict( - **NotificationAssociateMixin.targets, - **{'approval_notification': [ - 'notification_templates_approvals', - 'notification_template' - ] - }, - ) + targets = NotificationAssociateMixin.targets.copy() class WorkflowJobTemplateNotificationDisAssociation(NotificationAssociateMixin, CustomAction): resource = 'workflow_job_templates' action = 'disassociate' - targets = dict( - **NotificationAssociateMixin.targets, - **{'approval_notification': [ - 'notification_templates_approvals', - 'notification_template' - ] - }, - ) + targets = NotificationAssociateMixin.targets.copy() + + +WorkflowJobTemplateNotificationAssociation.targets.update({ + 'approval_notification': ['notification_templates_approvals', 'notification_template'], +}) +WorkflowJobTemplateNotificationDisAssociation.targets.update({ + 'approval_notification': ['notification_templates_approvals', 'notification_template'], +}) class ProjectNotificationAssociation(NotificationAssociateMixin, CustomAction): @@ -346,29 +340,22 @@ class InventorySourceNotificationDisAssociation(NotificationAssociateMixin, Cust class OrganizationNotificationAssociation(NotificationAssociateMixin, CustomAction): resource = 'organizations' action = 'associate' - targets = dict( - **NotificationAssociateMixin.targets, - **{'approval_notification': [ - 'notification_templates_approvals', - 'notification_template' - ] - }, - ) + targets = NotificationAssociateMixin.targets.copy() class OrganizationNotificationDisAssociation(NotificationAssociateMixin, CustomAction): resource = 'organizations' action = 'disassociate' - targets = dict( - **NotificationAssociateMixin.targets, - **{'approval_notification': [ - 'notification_templates_approvals', - 'notification_template' - ] - }, - ) + targets = NotificationAssociateMixin.targets.copy() +OrganizationNotificationAssociation.targets.update({ + 'approval_notification': ['notification_templates_approvals', 'notification_template'], +}) +OrganizationNotificationDisAssociation.targets.update({ + 'approval_notification': ['notification_templates_approvals', 'notification_template'], +}) + class SettingsList(CustomAction): action = 'list' resource = 'settings' From aa4f5ccca91f10ff1f9dfd460fa67d645c134c51 Mon Sep 17 00:00:00 2001 From: beeankha Date: Fri, 11 Oct 2019 11:38:47 -0400 Subject: [PATCH 26/27] Add blank line (flake8) --- awxkit/awxkit/cli/custom.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awxkit/awxkit/cli/custom.py b/awxkit/awxkit/cli/custom.py index f9077acab5..45146b493e 100644 --- a/awxkit/awxkit/cli/custom.py +++ b/awxkit/awxkit/cli/custom.py @@ -356,6 +356,7 @@ OrganizationNotificationDisAssociation.targets.update({ 'approval_notification': ['notification_templates_approvals', 'notification_template'], }) + class SettingsList(CustomAction): action = 'list' resource = 'settings' From 62e4ebb85d579f6a9330901760f3f90a92ccf3fd Mon Sep 17 00:00:00 2001 From: beeankha Date: Wed, 16 Oct 2019 09:29:50 -0400 Subject: [PATCH 27/27] Minor change to README, plus a rebase. --- awxkit/awxkit/cli/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/awxkit/cli/docs/README.md b/awxkit/awxkit/cli/docs/README.md index 11676ee5dd..d8f1dfc85a 100644 --- a/awxkit/awxkit/cli/docs/README.md +++ b/awxkit/awxkit/cli/docs/README.md @@ -1,7 +1,7 @@ AWX Command Line Interface ========================== -awx is the official command-line client for AWX. It: +`awx` is the official command-line client for AWX. It: * Uses naming and structure consistent with the AWX HTTP API * Provides consistent output formats with optional machine-parsable formats