Merge pull request #4658 from ryanpetrello/7.0.0-release

Bump VERSION to 7.0.0

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.gitignore

@@ -28,6 +28,9 @@ awx/ui/build_test
 awx/ui/client/languages
 awx/ui/templates/ui/index.html
 awx/ui/templates/ui/installing.html
+awx/ui_next/node_modules/
+awx/ui_next/coverage/
+awx/ui_next/build/locales/_build
 /tower-license
 /tower-license/**
 tools/prometheus/data
@@ -122,6 +125,7 @@ local/
 requirements/vendor
 .i18n_built
 .idea/*
+*credentials*.y*ml*
 
 # AWX python libs populated by requirements.txt
 awx/lib/.deps_built

CONTRIBUTING.md

@@ -134,6 +134,8 @@ Run the following to build the AWX UI:
 ```bash
 (host) $ make ui-devel
 ```
+See [the ui development documentation](awx/ui/README.md) for more information on using the frontend development, build, and test tooling.
+
 ### Running the environment
 
 #### Start the containers

INSTALL.md

@@ -36,7 +36,7 @@ This document provides a guide for installing AWX.
       - [PostgreSQL](#postgresql-1)
       - [Proxy settings](#proxy-settings)
   - [Start the build](#start-the-build-2)
-  - [Post build](#post-build-1)
+  - [Post build](#post-build-2)
   - [Accessing AWX](#accessing-awx-2)
 
 ## Getting started
@@ -72,7 +72,7 @@ Before you can run a deployment, you'll need the following installed in your loc
 
 The system that runs the AWX service will need to satisfy the following requirements
 
-- At leasts 4GB of memory
+- At least 4GB of memory
 - At least 2 cpu cores
 - At least 20GB of space
 - Running Docker, Openshift, or Kubernetes

Makefile

@@ -59,20 +59,23 @@ UI_RELEASE_FLAG_FILE = awx/ui/.release_built
 I18N_FLAG_FILE = .i18n_built
 
 .PHONY: awx-link clean clean-tmp clean-venv requirements requirements_dev \
-	develop refresh adduser migrate dbchange dbshell runserver \
+	develop refresh adduser migrate dbchange runserver \
 	receiver test test_unit test_coverage coverage_html \
 	dev_build release_build release_clean sdist \
 	ui-docker-machine ui-docker ui-release ui-devel \
 	ui-test ui-deps ui-test-ci VERSION
 
 # remove ui build artifacts
-clean-ui:
+clean-ui: clean-languages
 	rm -rf awx/ui/static/
 	rm -rf awx/ui/node_modules/
 	rm -rf awx/ui/test/unit/reports/
 	rm -rf awx/ui/test/spec/reports/
 	rm -rf awx/ui/test/e2e/reports/
 	rm -rf awx/ui/client/languages/
+	rm -rf awx/ui_next/node_modules/
+	rm -rf awx/ui_next/coverage/
+	rm -rf awx/ui_next/build/locales/_build/
 	rm -f $(UI_DEPS_FLAG_FILE)
 	rm -f $(UI_RELEASE_DEPS_FLAG_FILE)
 	rm -f $(UI_RELEASE_FLAG_FILE)
@@ -91,6 +94,10 @@ clean-schema:
 	rm -rf schema.json
 	rm -rf reference-schema.json
 
+clean-languages:
+	rm -f $(I18N_FLAG_FILE)
+	find . -type f -regex ".*\.mo$$" -delete
+
 # Remove temporary build files, compiled Python files.
 clean: clean-ui clean-dist
 	rm -rf awx/public
@@ -98,7 +105,7 @@ clean: clean-ui clean-dist
 	rm -rf awx/job_status
 	rm -rf awx/job_output
 	rm -rf reports
-	rm -f awx/awx_test.sqlite3
+	rm -f awx/awx_test.sqlite3*
 	rm -rf requirements/vendor
 	rm -rf tmp
 	rm -rf $(I18N_FLAG_FILE)
@@ -124,8 +131,8 @@ virtualenv_ansible:
 		if [ ! -d "$(VENV_BASE)/ansible" ]; then \
 			virtualenv -p python --system-site-packages $(VENV_BASE)/ansible && \
 			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed six packaging appdirs && \
-			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==41.0.1 && \
-			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==19.1.1; \
+			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==36.0.1 && \
+			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==9.0.1; \
 		fi; \
 	fi
 
@@ -239,10 +246,6 @@ migrate:
 dbchange:
 	$(MANAGEMENT_COMMAND) makemigrations
 
-# access database shell, asks for password
-dbshell:
-	sudo -u postgres psql -d awx-dev
-
 server_noattach:
 	tmux new-session -d -s awx 'exec make uwsgi'
 	tmux rename-window 'AWX'
@@ -369,6 +372,7 @@ test:
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
 	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider -n auto $(TEST_DIRS)
+	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py2,py3
 	awx-manage check_migrations --dry-run --check  -n 'vNNN_missing_migration_file'
 
 test_unit:
@@ -515,6 +519,21 @@ jshint: $(UI_DEPS_FLAG_FILE)
 # END UI TASKS
 # --------------------------------------
 
+# UI NEXT TASKS
+# --------------------------------------
+
+ui-next-lint:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next lint
+	$(NPM_BIN) run --prefix awx/ui_next prettier-check
+
+ui-next-test:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next test
+
+# END UI NEXT TASKS
+# --------------------------------------
+
 # Build a pip-installable package into dist/ with a timestamped version number.
 dev_build:
 	$(PYTHON) setup.py dev_build
@@ -546,8 +565,8 @@ setup-bundle-build:
 	mkdir -p $@
 
 docker-auth:
-	if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \
-		docker login -u oauth2accesstoken -p "$(IMAGE_REPOSITORY_AUTH)" $(IMAGE_REPOSITORY_BASE); \
+	@if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \
+		echo "$(IMAGE_REPOSITORY_AUTH)" | docker login -u oauth2accesstoken --password-stdin $(IMAGE_REPOSITORY_BASE); \
 	fi;
 
 # Docker isolated rampart

VERSION

@@ -1 +1 @@
-5.0.0
+7.0.0

awx/__init__.py

@@ -25,9 +25,8 @@ import hashlib
 
 try:
     import django
-    from django.utils.encoding import force_bytes
-    from django.db.backends.base.schema import BaseDatabaseSchemaEditor
     from django.db.backends.base import schema
+    from django.db.backends.utils import names_digest
     HAS_DJANGO = True
 except ImportError:
     HAS_DJANGO = False
@@ -37,30 +36,33 @@ if HAS_DJANGO is True:
     # This line exists to make sure we don't regress on FIPS support if we
     # upgrade Django; if you're upgrading Django and see this error,
     # update the version check below, and confirm that FIPS still works.
-    if django.__version__ != '1.11.20':
-        raise RuntimeError("Django version other than 1.11.20 detected {}. \
-                Subclassing BaseDatabaseSchemaEditor is known to work for Django 1.11.20 \
-                and may not work in newer Django versions.".format(django.__version__))
+    # If operating in a FIPS environment, `hashlib.md5()` will raise a `ValueError`,
+    # but will support the `usedforsecurity` keyword on RHEL and Centos systems.
 
+    # Keep an eye on https://code.djangoproject.com/ticket/28401
+    target_version = '2.2.4'
+    if django.__version__ != target_version:
+        raise RuntimeError(
+            "Django version other than {target} detected: {current}. "
+            "Overriding `names_digest` is known to work for Django {target} "
+            "and may not work in other Django versions.".format(target=target_version,
+                                                                current=django.__version__)
+        )
 
-    class FipsBaseDatabaseSchemaEditor(BaseDatabaseSchemaEditor):
-
-        @classmethod
-        def _digest(cls, *args):
+    try:
+        names_digest('foo', 'bar', 'baz', length=8)
+    except ValueError:
+        def names_digest(*args, length):
             """
-            Generates a 32-bit digest of a set of arguments that can be used to
-            shorten identifying names.
+            Generate a 32-bit digest of a set of arguments that can be used to shorten
+            identifying names.  Support for use in FIPS environments.
             """
-            try:
-                h = hashlib.md5()
-            except ValueError:
-                h = hashlib.md5(usedforsecurity=False)
+            h = hashlib.md5(usedforsecurity=False)
             for arg in args:
-                h.update(force_bytes(arg))
-            return h.hexdigest()[:8]
+                h.update(arg.encode())
+            return h.hexdigest()[:length]
 
-
-    schema.BaseDatabaseSchemaEditor = FipsBaseDatabaseSchemaEditor
+        schema.names_digest = names_digest
 
 
 def find_commands(management_dir):

awx/api/generics.py

@@ -34,7 +34,8 @@ from rest_framework.negotiation import DefaultContentNegotiation
 # AWX
 from awx.api.filters import FieldLookupBackend
 from awx.main.models import (
-    UnifiedJob, UnifiedJobTemplate, User, Role, Credential
+    UnifiedJob, UnifiedJobTemplate, User, Role, Credential,
+    WorkflowJobTemplateNode, WorkflowApprovalTemplate
 )
 from awx.main.access import access_registry
 from awx.main.utils import (
@@ -401,21 +402,21 @@ class ListAPIView(generics.ListAPIView, GenericAPIView):
                 continue
             if getattr(field, 'related_model', None):
                 fields.add('{}__search'.format(field.name))
-        for rel in self.model._meta.related_objects:
-            name = rel.related_name
-            if isinstance(rel, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
+        for related in self.model._meta.related_objects:
+            name = related.related_name
+            if isinstance(related, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
                 # Add underscores for polymorphic subclasses for user utility
-                name = rel.related_model._meta.verbose_name.replace(" ", "_")
+                name = related.related_model._meta.verbose_name.replace(" ", "_")
             if skip_related_name(name) or name.endswith('+'):
                 continue
             fields.add('{}__search'.format(name))
-        m2m_rel = []
-        m2m_rel += self.model._meta.local_many_to_many
+        m2m_related = []
+        m2m_related += self.model._meta.local_many_to_many
         if issubclass(self.model, UnifiedJobTemplate) and self.model != UnifiedJobTemplate:
-            m2m_rel += UnifiedJobTemplate._meta.local_many_to_many
+            m2m_related += UnifiedJobTemplate._meta.local_many_to_many
         if issubclass(self.model, UnifiedJob) and self.model != UnifiedJob:
-            m2m_rel += UnifiedJob._meta.local_many_to_many
-        for relationship in m2m_rel:
+            m2m_related += UnifiedJob._meta.local_many_to_many
+        for relationship in m2m_related:
             if skip_related_name(relationship.name):
                 continue
             if relationship.related_model._meta.app_label != 'main':
@@ -882,6 +883,21 @@ class CopyAPIView(GenericAPIView):
                 create_kwargs[field.name] = CopyAPIView._decrypt_model_field_if_needed(
                     obj, field.name, field_val
                 )
+
+        # WorkflowJobTemplateNodes that represent an approval are *special*;
+        # when we copy them, we actually want to *copy* the UJT they point at
+        # rather than share the template reference between nodes in disparate
+        # workflows
+        if (
+            isinstance(obj, WorkflowJobTemplateNode) and
+            isinstance(getattr(obj, 'unified_job_template'), WorkflowApprovalTemplate)
+        ):
+            new_approval_template, sub_objs = CopyAPIView.copy_model_obj(
+                None, None, WorkflowApprovalTemplate,
+                obj.unified_job_template, creater
+            )
+            create_kwargs['unified_job_template'] = new_approval_template
+
         new_obj = model.objects.create(**create_kwargs)
         logger.debug('Deep copy: Created new object {}({})'.format(
             new_obj, model

awx/api/metadata.py

@@ -5,6 +5,8 @@ from collections import OrderedDict
 
 # Django
 from django.core.exceptions import PermissionDenied
+from django.db.models.fields import PositiveIntegerField, BooleanField
+from django.db.models.fields.related import ForeignKey
 from django.http import Http404
 from django.utils.encoding import force_text, smart_text
 from django.utils.translation import ugettext_lazy as _
@@ -14,9 +16,11 @@ from rest_framework import exceptions
 from rest_framework import metadata
 from rest_framework import serializers
 from rest_framework.relations import RelatedField, ManyRelatedField
+from rest_framework.fields import JSONField as DRFJSONField
 from rest_framework.request import clone_request
 
 # AWX
+from awx.main.fields import JSONField
 from awx.main.models import InventorySource, NotificationTemplate
 
 
@@ -68,6 +72,8 @@ class Metadata(metadata.SimpleMetadata):
             else:
                 for model_field in serializer.Meta.model._meta.fields:
                     if field.field_name == model_field.name:
+                        if getattr(model_field, '__accepts_json__', None):
+                            field_info['type'] = 'json'
                         field_info['filterable'] = True
                         break
                 else:
@@ -114,15 +120,48 @@ class Metadata(metadata.SimpleMetadata):
             for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES:
                 field_info[notification_type_name] = notification_type_class.init_parameters
 
+        # Special handling of notification messages where the required properties
+        # are conditional on the type selected.
+        try:
+            view_model = field.context['view'].model
+        except (AttributeError, KeyError):
+            view_model = None
+        if view_model == NotificationTemplate and field.field_name == 'messages':
+            for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES:
+                field_info[notification_type_name] = notification_type_class.default_messages
+
+
         # Update type of fields returned...
+        model_field = None
+        if serializer and hasattr(serializer, 'Meta') and hasattr(serializer.Meta, 'model'):
+            try:
+                model_field = serializer.Meta.model._meta.get_field(field.field_name)
+            except Exception:
+                pass
         if field.field_name == 'type':
             field_info['type'] = 'choice'
-        elif field.field_name == 'url':
+        elif field.field_name in ('url', 'custom_virtualenv', 'token'):
             field_info['type'] = 'string'
         elif field.field_name in ('related', 'summary_fields'):
             field_info['type'] = 'object'
+        elif isinstance(field, PositiveIntegerField):
+            field_info['type'] = 'integer'
         elif field.field_name in ('created', 'modified'):
             field_info['type'] = 'datetime'
+        elif (
+            RelatedField in field.__class__.__bases__ or
+            isinstance(model_field, ForeignKey)
+        ):
+            field_info['type'] = 'id'
+        elif (
+            isinstance(field, JSONField) or
+            isinstance(model_field, JSONField) or
+            isinstance(field, DRFJSONField) or
+            isinstance(getattr(field, 'model_field', None), JSONField)
+        ):
+            field_info['type'] = 'json'
+        elif isinstance(model_field, BooleanField):
+            field_info['type'] = 'boolean'
 
         return field_info
 

awx/api/permissions.py

@@ -17,7 +17,7 @@ logger = logging.getLogger('awx.api.permissions')
 
 __all__ = ['ModelAccessPermission', 'JobTemplateCallbackPermission', 'VariableDataPermission',
            'TaskPermission', 'ProjectUpdatePermission', 'InventoryInventorySourcesUpdatePermission',
-           'UserPermission', 'IsSuperUser', 'InstanceGroupTowerPermission',]
+           'UserPermission', 'IsSuperUser', 'InstanceGroupTowerPermission', 'WorkflowApprovalPermission']
 
 
 class ModelAccessPermission(permissions.BasePermission):
@@ -95,7 +95,7 @@ class ModelAccessPermission(permissions.BasePermission):
         '''
 
         # Don't allow anonymous users. 401, not 403, hence no raised exception.
-        if not request.user or request.user.is_anonymous():
+        if not request.user or request.user.is_anonymous:
             return False
 
         # Always allow superusers
@@ -196,6 +196,17 @@ class TaskPermission(ModelAccessPermission):
             return False
 
 
+class WorkflowApprovalPermission(ModelAccessPermission):
+    '''
+    Permission check used by workflow `approval` and `deny` views to determine
+    who has access to approve and deny paused workflow nodes
+    '''
+
+    def check_post_permissions(self, request, view, obj=None):
+        approval = get_object_or_400(view.model, pk=view.kwargs['pk'])
+        return check_user_access(request.user, view.model, 'approve_or_deny', approval)
+
+
 class ProjectUpdatePermission(ModelAccessPermission):
     '''
     Permission check used by ProjectUpdateView to determine who can update projects
@@ -238,4 +249,3 @@ class InstanceGroupTowerPermission(ModelAccessPermission):
         if request.method == 'DELETE' and obj.name == "tower":
             return False
         return super(InstanceGroupTowerPermission, self).has_object_permission(request, view, obj)
-

awx/api/renderers.py

@@ -2,6 +2,7 @@
 # All Rights Reserved.
 
 from django.utils.safestring import SafeText
+from prometheus_client.parser import text_string_to_metric_families
 
 # Django REST Framework
 from rest_framework import renderers
@@ -103,3 +104,21 @@ class AnsiTextRenderer(PlainTextRenderer):
 class AnsiDownloadRenderer(PlainTextRenderer):
 
     format = "ansi_download"
+
+
+class PrometheusJSONRenderer(renderers.JSONRenderer):
+
+    def render(self, data, accepted_media_type=None, renderer_context=None):
+        if isinstance(data, dict):
+            # HTTP errors are {'detail': ErrorDetail(string='...', code=...)}
+            return super(PrometheusJSONRenderer, self).render(
+                data, accepted_media_type, renderer_context
+            )
+        parsed_metrics = text_string_to_metric_families(data)
+        data = {}
+        for family in parsed_metrics:
+            for sample in family.samples:
+                data[sample[0]] = {"labels": sample[1], "value": sample[2]}
+        return super(PrometheusJSONRenderer, self).render(
+            data, accepted_media_type, renderer_context
+        )

awx/api/serializers.py

@@ -13,6 +13,10 @@ from datetime import timedelta
 from oauthlib import oauth2
 from oauthlib.common import generate_token
 
+# Jinja
+from jinja2 import sandbox, StrictUndefined
+from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
+
 # Django
 from django.conf import settings
 from django.contrib.auth import update_session_auth_hash
@@ -50,12 +54,12 @@ from awx.main.models import (
     CredentialType, CustomInventoryScript, Group, Host, Instance,
     InstanceGroup, Inventory, InventorySource, InventoryUpdate,
     InventoryUpdateEvent, Job, JobEvent, JobHostSummary, JobLaunchConfig,
-    JobTemplate, Label, Notification, NotificationTemplate,
+    JobNotificationMixin, JobTemplate, Label, Notification, NotificationTemplate,
     OAuth2AccessToken, OAuth2Application, Organization, Project,
     ProjectUpdate, ProjectUpdateEvent, RefreshToken, Role, Schedule,
     SystemJob, SystemJobEvent, SystemJobTemplate, Team, UnifiedJob,
-    UnifiedJobTemplate, WorkflowJob, WorkflowJobNode,
-    WorkflowJobTemplate, WorkflowJobTemplateNode, StdoutMaxBytesExceeded
+    UnifiedJobTemplate, WorkflowApproval, WorkflowApprovalTemplate, WorkflowJob,
+    WorkflowJobNode, WorkflowJobTemplate, WorkflowJobTemplateNode, StdoutMaxBytesExceeded
 )
 from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES
 from awx.main.models.rbac import (
@@ -117,6 +121,8 @@ SUMMARIZABLE_FK_FIELDS = {
     'job_template': DEFAULT_SUMMARY_FIELDS,
     'workflow_job_template': DEFAULT_SUMMARY_FIELDS,
     'workflow_job': DEFAULT_SUMMARY_FIELDS,
+    'workflow_approval_template': DEFAULT_SUMMARY_FIELDS + ('timeout',),
+    'workflow_approval': DEFAULT_SUMMARY_FIELDS + ('timeout',),
     'schedule': DEFAULT_SUMMARY_FIELDS + ('next_run',),
     'unified_job_template': DEFAULT_SUMMARY_FIELDS + ('unified_job_type',),
     'last_job': DEFAULT_SUMMARY_FIELDS + ('finished', 'status', 'failed', 'license_error'),
@@ -677,6 +683,8 @@ class UnifiedJobTemplateSerializer(BaseSerializer):
                 serializer_class = SystemJobTemplateSerializer
             elif isinstance(obj, WorkflowJobTemplate):
                 serializer_class = WorkflowJobTemplateSerializer
+            elif isinstance(obj, WorkflowApprovalTemplate):
+                serializer_class = WorkflowApprovalTemplateSerializer
         return serializer_class
 
     def to_representation(self, obj):
@@ -778,6 +786,8 @@ class UnifiedJobSerializer(BaseSerializer):
                 serializer_class = SystemJobSerializer
             elif isinstance(obj, WorkflowJob):
                 serializer_class = WorkflowJobSerializer
+            elif isinstance(obj, WorkflowApproval):
+                serializer_class = WorkflowApprovalSerializer
         return serializer_class
 
     def to_representation(self, obj):
@@ -834,6 +844,8 @@ class UnifiedJobListSerializer(UnifiedJobSerializer):
                 serializer_class = SystemJobListSerializer
             elif isinstance(obj, WorkflowJob):
                 serializer_class = WorkflowJobListSerializer
+            elif isinstance(obj, WorkflowApproval):
+                serializer_class = WorkflowApprovalListSerializer
         return serializer_class
 
     def to_representation(self, obj):
@@ -1246,7 +1258,7 @@ class OrganizationSerializer(BaseSerializer):
             applications    = self.reverse('api:organization_applications_list',     kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:organization_activity_stream_list', kwargs={'pk': obj.pk}),
             notification_templates = self.reverse('api:organization_notification_templates_list', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:organization_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:organization_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:organization_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:organization_notification_templates_error_list', kwargs={'pk': obj.pk}),
             object_roles = self.reverse('api:organization_object_roles_list', kwargs={'pk': obj.pk}),
@@ -1285,8 +1297,8 @@ class OrganizationSerializer(BaseSerializer):
 class ProjectOptionsSerializer(BaseSerializer):
 
     class Meta:
-        fields = ('*', 'local_path', 'scm_type', 'scm_url', 'scm_branch',
-                  'scm_clean', 'scm_delete_on_update', 'credential', 'timeout',)
+        fields = ('*', 'local_path', 'scm_type', 'scm_url', 'scm_branch', 'scm_refspec',
+                  'scm_clean', 'scm_delete_on_update', 'credential', 'timeout', 'scm_revision')
 
     def get_related(self, obj):
         res = super(ProjectOptionsSerializer, self).get_related(obj)
@@ -1311,18 +1323,14 @@ class ProjectOptionsSerializer(BaseSerializer):
             attrs.pop('local_path', None)
         if 'local_path' in attrs and attrs['local_path'] not in valid_local_paths:
             errors['local_path'] = _('This path is already being used by another manual project.')
+        if attrs.get('scm_refspec') and scm_type != 'git':
+            errors['scm_refspec'] = _('SCM refspec can only be used with git projects.')
 
         if errors:
             raise serializers.ValidationError(errors)
 
         return super(ProjectOptionsSerializer, self).validate(attrs)
 
-    def to_representation(self, obj):
-        ret = super(ProjectOptionsSerializer, self).to_representation(obj)
-        if obj is not None and 'credential' in ret and not obj.credential:
-            ret['credential'] = None
-        return ret
-
 
 class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
 
@@ -1338,7 +1346,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
     class Meta:
         model = Project
         fields = ('*', 'organization', 'scm_update_on_launch',
-                  'scm_update_cache_timeout', 'scm_revision', 'custom_virtualenv',) + \
+                  'scm_update_cache_timeout', 'allow_override', 'custom_virtualenv',) + \
                  ('last_update_failed', 'last_updated')  # Backwards compatibility
 
     def get_related(self, obj):
@@ -1352,7 +1360,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
             scm_inventory_sources = self.reverse('api:project_scm_inventory_sources', kwargs={'pk': obj.pk}),
             schedules = self.reverse('api:project_schedules_list', kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:project_activity_stream_list', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:project_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:project_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:project_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:project_notification_templates_error_list', kwargs={'pk': obj.pk}),
             access_list = self.reverse('api:project_access_list', kwargs={'pk': obj.pk}),
@@ -1388,6 +1396,21 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
         elif self.instance:
             organization = self.instance.organization
 
+        if 'allow_override' in attrs and self.instance:
+            # case where user is turning off this project setting
+            if self.instance.allow_override and not attrs['allow_override']:
+                used_by = set(
+                    JobTemplate.objects.filter(
+                        models.Q(project=self.instance),
+                        models.Q(ask_scm_branch_on_launch=True) | ~models.Q(scm_branch="")
+                    ).values_list('pk', flat=True)
+                )
+                if used_by:
+                    raise serializers.ValidationError({
+                        'allow_override': _('One or more job templates depend on branch override behavior for this project (ids: {}).').format(
+                            ' '.join([str(pk) for pk in used_by])
+                        )})
+
         view = self.context.get('view', None)
         if not organization and not view.request.user.is_superuser:
             # Only allow super users to create orgless projects
@@ -1943,6 +1966,25 @@ class InventorySourceOptionsSerializer(BaseSerializer):
 
         return super(InventorySourceOptionsSerializer, self).validate(attrs)
 
+    # TODO: remove when old 'credential' fields are removed
+    def get_summary_fields(self, obj):
+        summary_fields = super(InventorySourceOptionsSerializer, self).get_summary_fields(obj)
+        all_creds = []
+        if 'credential' in summary_fields:
+            cred = obj.get_cloud_credential()
+            if cred:
+                summarized_cred = {
+                    'id': cred.id, 'name': cred.name, 'description': cred.description,
+                    'kind': cred.kind, 'cloud': True
+                }
+                summary_fields['credential'] = summarized_cred
+                all_creds.append(summarized_cred)
+                summary_fields['credential']['credential_type_id'] = cred.credential_type_id
+            else:
+                summary_fields.pop('credential')
+        summary_fields['credentials'] = all_creds
+        return summary_fields
+
 
 class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOptionsSerializer):
 
@@ -1960,6 +2002,9 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
         fields = ('*', 'name', 'inventory', 'update_on_launch', 'update_cache_timeout',
                   'source_project', 'update_on_project_update') + \
                  ('last_update_failed', 'last_updated') # Backwards compatibility.
+        extra_kwargs = {
+            'inventory': {'required': True}
+        }
 
     def get_related(self, obj):
         res = super(InventorySourceSerializer, self).get_related(obj)
@@ -1970,7 +2015,7 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
             activity_stream = self.reverse('api:inventory_source_activity_stream_list', kwargs={'pk': obj.pk}),
             hosts = self.reverse('api:inventory_source_hosts_list', kwargs={'pk': obj.pk}),
             groups = self.reverse('api:inventory_source_groups_list', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:inventory_source_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:inventory_source_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:inventory_source_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:inventory_source_notification_templates_error_list', kwargs={'pk': obj.pk}),
         ))
@@ -2541,7 +2586,7 @@ class CredentialSerializer(BaseSerializer):
 
     def validate_credential_type(self, credential_type):
         if self.instance and credential_type.pk != self.instance.credential_type.pk:
-            for rel in (
+            for related_objects in (
                 'ad_hoc_commands',
                 'insights_inventories',
                 'unifiedjobs',
@@ -2550,7 +2595,7 @@ class CredentialSerializer(BaseSerializer):
                 'projectupdates',
                 'workflowjobnodes'
             ):
-                if getattr(self.instance, rel).count() > 0:
+                if getattr(self.instance, related_objects).count() > 0:
                     raise ValidationError(
                         _('You cannot change the credential type of the credential, as it may break the functionality'
                           ' of the resources using it.'),
@@ -2682,7 +2727,7 @@ class LabelsListMixin(object):
 class JobOptionsSerializer(LabelsListMixin, BaseSerializer):
 
     class Meta:
-        fields = ('*', 'job_type', 'inventory', 'project', 'playbook',
+        fields = ('*', 'job_type', 'inventory', 'project', 'playbook', 'scm_branch',
                   'forks', 'limit', 'verbosity', 'extra_vars', 'job_tags',
                   'force_handlers', 'skip_tags', 'start_at_task', 'timeout',
                   'use_fact_cache',)
@@ -2729,16 +2774,28 @@ class JobOptionsSerializer(LabelsListMixin, BaseSerializer):
 
     def validate(self, attrs):
         if 'project' in self.fields and 'playbook' in self.fields:
-            project = attrs.get('project', self.instance and self.instance.project or None)
+            project = attrs.get('project', self.instance.project if self.instance else None)
             playbook = attrs.get('playbook', self.instance and self.instance.playbook or '')
+            scm_branch = attrs.get('scm_branch', self.instance.scm_branch if self.instance else None)
+            ask_scm_branch_on_launch = attrs.get(
+                'ask_scm_branch_on_launch', self.instance.ask_scm_branch_on_launch if self.instance else None)
             if not project:
                 raise serializers.ValidationError({'project': _('This field is required.')})
-            if project and project.scm_type and playbook and force_text(playbook) not in project.playbook_files:
-                raise serializers.ValidationError({'playbook': _('Playbook not found for project.')})
-            if project and not project.scm_type and playbook and force_text(playbook) not in project.playbooks:
+            playbook_not_found = bool(
+                (
+                    project and project.scm_type and (not project.allow_override) and
+                    playbook and force_text(playbook) not in project.playbook_files
+                ) or
+                (project and not project.scm_type and playbook and force_text(playbook) not in project.playbooks)  # manual
+            )
+            if playbook_not_found:
                 raise serializers.ValidationError({'playbook': _('Playbook not found for project.')})
             if project and not playbook:
                 raise serializers.ValidationError({'playbook': _('Must select playbook for project.')})
+            if scm_branch and not project.allow_override:
+                raise serializers.ValidationError({'scm_branch': _('Project does not allow overriding branch.')})
+            if ask_scm_branch_on_launch and not project.allow_override:
+                raise serializers.ValidationError({'ask_scm_branch_on_launch': _('Project does not allow overriding branch.')})
 
         ret = super(JobOptionsSerializer, self).validate(attrs)
         return ret
@@ -2780,7 +2837,8 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
 
     class Meta:
         model = JobTemplate
-        fields = ('*', 'host_config_key', 'ask_diff_mode_on_launch', 'ask_variables_on_launch', 'ask_limit_on_launch', 'ask_tags_on_launch',
+        fields = ('*', 'host_config_key', 'ask_scm_branch_on_launch', 'ask_diff_mode_on_launch', 'ask_variables_on_launch',
+                  'ask_limit_on_launch', 'ask_tags_on_launch',
                   'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_verbosity_on_launch', 'ask_inventory_on_launch',
                   'ask_credential_on_launch', 'survey_enabled', 'become_enabled', 'diff_mode',
                   'allow_simultaneous', 'custom_virtualenv', 'job_slice_count')
@@ -2792,7 +2850,7 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
             schedules = self.reverse('api:job_template_schedules_list', kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:job_template_activity_stream_list', kwargs={'pk': obj.pk}),
             launch = self.reverse('api:job_template_launch', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:job_template_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
             access_list = self.reverse('api:job_template_access_list',      kwargs={'pk': obj.pk}),
@@ -3204,7 +3262,7 @@ class SystemJobTemplateSerializer(UnifiedJobTemplateSerializer):
             jobs = self.reverse('api:system_job_template_jobs_list', kwargs={'pk': obj.pk}),
             schedules = self.reverse('api:system_job_template_schedules_list', kwargs={'pk': obj.pk}),
             launch = self.reverse('api:system_job_template_launch', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:system_job_template_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:system_job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:system_job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:system_job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
 
@@ -3271,7 +3329,7 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
             workflow_nodes = self.reverse('api:workflow_job_template_workflow_nodes_list', kwargs={'pk': obj.pk}),
             labels = self.reverse('api:workflow_job_template_label_list', kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:workflow_job_template_activity_stream_list', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:workflow_job_template_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:workflow_job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:workflow_job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:workflow_job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
             access_list = self.reverse('api:workflow_job_template_access_list', kwargs={'pk': obj.pk}),
@@ -3345,7 +3403,78 @@ class WorkflowJobCancelSerializer(WorkflowJobSerializer):
         fields = ('can_cancel',)
 
 
+class WorkflowApprovalViewSerializer(UnifiedJobSerializer):
+
+    class Meta:
+        model = WorkflowApproval
+        fields = []
+
+
+class WorkflowApprovalSerializer(UnifiedJobSerializer):
+
+    can_approve_or_deny = serializers.SerializerMethodField()
+    approval_expiration = serializers.SerializerMethodField()
+    timed_out = serializers.ReadOnlyField()
+
+    class Meta:
+        model = WorkflowApproval
+        fields = ('*', '-controller_node', '-execution_node', 'can_approve_or_deny', 'approval_expiration', 'timed_out',)
+
+    def get_approval_expiration(self, obj):
+        if obj.status != 'pending' or obj.timeout == 0:
+            return None
+        return obj.created + timedelta(seconds=obj.timeout)
+
+    def get_can_approve_or_deny(self, obj):
+        request = self.context.get('request', None)
+        allowed = request.user.can_access(WorkflowApproval, 'approve_or_deny', obj)
+        return allowed is True and obj.status == 'pending'
+
+    def get_related(self, obj):
+        res = super(WorkflowApprovalSerializer, self).get_related(obj)
+
+        if obj.workflow_approval_template:
+            res['workflow_approval_template'] = self.reverse('api:workflow_approval_template_detail',
+                                                             kwargs={'pk': obj.workflow_approval_template.pk})
+        res['approve'] = self.reverse('api:workflow_approval_approve', kwargs={'pk': obj.pk})
+        res['deny'] = self.reverse('api:workflow_approval_deny', kwargs={'pk': obj.pk})
+        return res
+
+
+class WorkflowApprovalActivityStreamSerializer(WorkflowApprovalSerializer):
+    """
+    timed_out and status are usually read-only fields
+    However, when we generate an activity stream record, we *want* to record
+    these types of changes.  This serializer allows us to do so.
+    """
+    status = serializers.ChoiceField(choices=JobTemplate.JOB_TEMPLATE_STATUS_CHOICES)
+    timed_out = serializers.BooleanField()
+
+
+
+class WorkflowApprovalListSerializer(WorkflowApprovalSerializer, UnifiedJobListSerializer):
+
+    class Meta:
+        fields = ('*', '-controller_node', '-execution_node', 'can_approve_or_deny', 'approval_expiration', 'timed_out',)
+
+
+class WorkflowApprovalTemplateSerializer(UnifiedJobTemplateSerializer):
+
+    class Meta:
+        model = WorkflowApprovalTemplate
+        fields = ('*', 'timeout', 'name',)
+
+    def get_related(self, obj):
+        res = super(WorkflowApprovalTemplateSerializer, self).get_related(obj)
+        if 'last_job' in res:
+            del res['last_job']
+
+        res.update(dict(jobs = self.reverse('api:workflow_approval_template_jobs_list', kwargs={'pk': obj.pk}),))
+        return res
+
+
 class LaunchConfigurationBaseSerializer(BaseSerializer):
+    scm_branch = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
     job_type = serializers.ChoiceField(allow_blank=True, allow_null=True, required=False, default=None,
                                        choices=NEW_JOB_TYPE_CHOICES)
     job_tags = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
@@ -3358,7 +3487,7 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
 
     class Meta:
         fields = ('*', 'extra_data', 'inventory', # Saved launch-time config fields
-                  'job_type', 'job_tags', 'skip_tags', 'limit', 'skip_tags', 'diff_mode', 'verbosity')
+                  'scm_branch', 'job_type', 'job_tags', 'skip_tags', 'limit', 'skip_tags', 'diff_mode', 'verbosity')
 
     def get_related(self, obj):
         res = super(LaunchConfigurationBaseSerializer, self).get_related(obj)
@@ -3390,12 +3519,6 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
             ret['extra_data'] = obj.display_extra_vars()
         return ret
 
-    def get_summary_fields(self, obj):
-        summary_fields = super(LaunchConfigurationBaseSerializer, self).get_summary_fields(obj)
-        # Credential would be an empty dictionary in this case
-        summary_fields.pop('credential', None)
-        return summary_fields
-
     def validate(self, attrs):
         db_extra_data = {}
         if self.instance:
@@ -3408,6 +3531,10 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
             ujt = attrs['unified_job_template']
         elif self.instance:
             ujt = self.instance.unified_job_template
+        if ujt is None:
+            if 'workflow_job_template' in attrs:
+                return {'workflow_job_template': attrs['workflow_job_template']}
+            return {}
 
         # build additional field survey_passwords to track redacted variables
         password_dict = {}
@@ -3477,7 +3604,6 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
 
 
 class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer):
-    credential = DeprecatedCredentialField()
     success_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     failure_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     always_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
@@ -3485,11 +3611,12 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer):
 
     class Meta:
         model = WorkflowJobTemplateNode
-        fields = ('*', 'credential', 'workflow_job_template', '-name', '-description', 'id', 'url', 'related',
+        fields = ('*', 'workflow_job_template', '-name', '-description', 'id', 'url', 'related',
                   'unified_job_template', 'success_nodes', 'failure_nodes', 'always_nodes',)
 
     def get_related(self, obj):
         res = super(WorkflowJobTemplateNodeSerializer, self).get_related(obj)
+        res['create_approval_template'] = self.reverse('api:workflow_job_template_node_create_approval', kwargs={'pk': obj.pk})
         res['success_nodes'] = self.reverse('api:workflow_job_template_node_success_nodes_list', kwargs={'pk': obj.pk})
         res['failure_nodes'] = self.reverse('api:workflow_job_template_node_failure_nodes_list', kwargs={'pk': obj.pk})
         res['always_nodes'] = self.reverse('api:workflow_job_template_node_always_nodes_list', kwargs={'pk': obj.pk})
@@ -3501,14 +3628,6 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer):
             pass
         return res
 
-    def build_field(self, field_name, info, model_class, nested_depth):
-        # have to special-case the field so that DRF will not automagically make it
-        # read-only because it's a property on the model.
-        if field_name == 'credential':
-            return self.build_standard_field(field_name,
-                                             self.credential)
-        return super(WorkflowJobTemplateNodeSerializer, self).build_field(field_name, info, model_class, nested_depth)
-
     def build_relational_field(self, field_name, relation_info):
         field_class, field_kwargs = super(WorkflowJobTemplateNodeSerializer, self).build_relational_field(field_name, relation_info)
         # workflow_job_template is read-only unless creating a new node.
@@ -3517,65 +3636,21 @@ class WorkflowJobTemplateNodeSerializer(LaunchConfigurationBaseSerializer):
             field_kwargs.pop('queryset', None)
         return field_class, field_kwargs
 
-    def validate(self, attrs):
-        deprecated_fields = {}
-        if 'credential' in attrs:  # TODO: remove when v2 API is deprecated
-            deprecated_fields['credential'] = attrs.pop('credential')
-        view = self.context.get('view')
-        attrs = super(WorkflowJobTemplateNodeSerializer, self).validate(attrs)
-        ujt_obj = None
-        if 'unified_job_template' in attrs:
-            ujt_obj = attrs['unified_job_template']
-        elif self.instance:
-            ujt_obj = self.instance.unified_job_template
-        if 'credential' in deprecated_fields:  # TODO: remove when v2 API is deprecated
-            cred = deprecated_fields['credential']
-            attrs['credential'] = cred
-            if cred is not None:
-                if not ujt_obj.ask_credential_on_launch:
-                    raise serializers.ValidationError({"credential": _(
-                        "Related template is not configured to accept credentials on launch.")})
-                cred = Credential.objects.get(pk=cred)
-                view = self.context.get('view', None)
-                if (not view) or (not view.request) or (view.request.user not in cred.use_role):
-                    raise PermissionDenied()
-        return attrs
-
-    def create(self, validated_data):  # TODO: remove when v2 API is deprecated
-        deprecated_fields = {}
-        if 'credential' in validated_data:
-            deprecated_fields['credential'] = validated_data.pop('credential')
-        obj = super(WorkflowJobTemplateNodeSerializer, self).create(validated_data)
-        if 'credential' in deprecated_fields:
-            if deprecated_fields['credential']:
-                obj.credentials.add(deprecated_fields['credential'])
-        return obj
-
-    def update(self, obj, validated_data):  # TODO: remove when v2 API is deprecated
-        deprecated_fields = {}
-        if 'credential' in validated_data:
-            deprecated_fields['credential'] = validated_data.pop('credential')
-        obj = super(WorkflowJobTemplateNodeSerializer, self).update(obj, validated_data)
-        if 'credential' in deprecated_fields:
-            existing = obj.credentials.filter(credential_type__kind='ssh')
-            new_cred = deprecated_fields['credential']
-            if new_cred not in existing:
-                for cred in existing:
-                    obj.credentials.remove(cred)
-                if new_cred:
-                    obj.credentials.add(new_cred)
-        return obj
+    def get_summary_fields(self, obj):
+        summary_fields = super(WorkflowJobTemplateNodeSerializer, self).get_summary_fields(obj)
+        if isinstance(obj.unified_job_template, WorkflowApprovalTemplate):
+            summary_fields['unified_job_template']['timeout'] = obj.unified_job_template.timeout
+        return summary_fields
 
 
 class WorkflowJobNodeSerializer(LaunchConfigurationBaseSerializer):
-    credential = DeprecatedCredentialField()
     success_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     failure_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
     always_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
 
     class Meta:
         model = WorkflowJobNode
-        fields = ('*', 'credential', 'job', 'workflow_job', '-name', '-description', 'id', 'url', 'related',
+        fields = ('*', 'job', 'workflow_job', '-name', '-description', 'id', 'url', 'related',
                   'unified_job_template', 'success_nodes', 'failure_nodes', 'always_nodes',
                   'do_not_run',)
 
@@ -3592,6 +3667,12 @@ class WorkflowJobNodeSerializer(LaunchConfigurationBaseSerializer):
             res['workflow_job'] = self.reverse('api:workflow_job_detail', kwargs={'pk': obj.workflow_job.pk})
         return res
 
+    def get_summary_fields(self, obj):
+        summary_fields = super(WorkflowJobNodeSerializer, self).get_summary_fields(obj)
+        if isinstance(obj.job, WorkflowApproval):
+            summary_fields['job']['timed_out'] = obj.job.timed_out
+        return summary_fields
+
 
 class WorkflowJobNodeListSerializer(WorkflowJobNodeSerializer):
     pass
@@ -3617,6 +3698,16 @@ class WorkflowJobTemplateNodeDetailSerializer(WorkflowJobTemplateNodeSerializer)
         return field_class, field_kwargs
 
 
+class WorkflowJobTemplateNodeCreateApprovalSerializer(BaseSerializer):
+
+    class Meta:
+        model = WorkflowApprovalTemplate
+        fields = ('timeout', 'name', 'description',)
+
+    def to_representation(self, obj):
+        return {}
+
+
 class JobListSerializer(JobSerializer, UnifiedJobListSerializer):
     pass
 
@@ -3941,6 +4032,7 @@ class JobLaunchSerializer(BaseSerializer):
         required=False, write_only=True
     )
     credential_passwords = VerbatimField(required=False, write_only=True)
+    scm_branch = serializers.CharField(required=False, write_only=True, allow_blank=True)
     diff_mode = serializers.BooleanField(required=False, write_only=True)
     job_tags = serializers.CharField(required=False, write_only=True, allow_blank=True)
     job_type = serializers.ChoiceField(required=False, choices=NEW_JOB_TYPE_CHOICES, write_only=True)
@@ -3951,13 +4043,15 @@ class JobLaunchSerializer(BaseSerializer):
     class Meta:
         model = JobTemplate
         fields = ('can_start_without_user_input', 'passwords_needed_to_start',
-                  'extra_vars', 'inventory', 'limit', 'job_tags', 'skip_tags', 'job_type', 'verbosity', 'diff_mode',
-                  'credentials', 'credential_passwords', 'ask_variables_on_launch', 'ask_tags_on_launch',
+                  'extra_vars', 'inventory', 'scm_branch', 'limit', 'job_tags', 'skip_tags', 'job_type', 'verbosity', 'diff_mode',
+                  'credentials', 'credential_passwords',
+                  'ask_scm_branch_on_launch', 'ask_variables_on_launch', 'ask_tags_on_launch',
                   'ask_diff_mode_on_launch', 'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_limit_on_launch',
                   'ask_verbosity_on_launch', 'ask_inventory_on_launch', 'ask_credential_on_launch',
                   'survey_enabled', 'variables_needed_to_start', 'credential_needed_to_start',
                   'inventory_needed_to_start', 'job_template_data', 'defaults', 'verbosity')
         read_only_fields = (
+            'ask_scm_branch_on_launch',
             'ask_diff_mode_on_launch', 'ask_variables_on_launch', 'ask_limit_on_launch', 'ask_tags_on_launch',
             'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_verbosity_on_launch',
             'ask_inventory_on_launch', 'ask_credential_on_launch',)
@@ -4143,7 +4237,8 @@ class NotificationTemplateSerializer(BaseSerializer):
 
     class Meta:
         model = NotificationTemplate
-        fields = ('*', 'organization', 'notification_type', 'notification_configuration')
+        fields = ('*', 'organization', 'notification_type', 'notification_configuration', 'messages')
+
 
     type_map = {"string": (str,),
                 "int": (int,),
@@ -4177,6 +4272,96 @@ class NotificationTemplateSerializer(BaseSerializer):
         d['recent_notifications'] = self._recent_notifications(obj)
         return d
 
+    def validate_messages(self, messages):
+        if messages is None:
+            return None
+
+        error_list = []
+        collected_messages = []
+
+        # Validate structure / content types
+        if not isinstance(messages, dict):
+            error_list.append(_("Expected dict for 'messages' field, found {}".format(type(messages))))
+        else:
+            for event in messages:
+                if event not in ['started', 'success', 'error']:
+                    error_list.append(_("Event '{}' invalid, must be one of 'started', 'success', or 'error'").format(event))
+                    continue
+                event_messages = messages[event]
+                if event_messages is None:
+                    continue
+                if not isinstance(event_messages, dict):
+                    error_list.append(_("Expected dict for event '{}', found {}").format(event, type(event_messages)))
+                    continue
+                for message_type in event_messages:
+                    if message_type not in ['message', 'body']:
+                        error_list.append(_("Message type '{}' invalid, must be either 'message' or 'body'").format(message_type))
+                        continue
+                    message = event_messages[message_type]
+                    if message is None:
+                        continue
+                    if not isinstance(message, str):
+                        error_list.append(_("Expected string for '{}', found {}, ").format(message_type, type(message)))
+                        continue
+                    if message_type == 'message':
+                        if '\n' in message:
+                            error_list.append(_("Messages cannot contain newlines (found newline in {} event)".format(event)))
+                            continue
+                    collected_messages.append(message)
+
+        # Subclass to return name of undefined field
+        class DescriptiveUndefined(StrictUndefined):
+            # The parent class prevents _accessing attributes_ of an object
+            # but will render undefined objects with 'Undefined'. This
+            # prevents their use entirely.
+            __repr__ = __str__ = StrictUndefined._fail_with_undefined_error
+
+            def __init__(self, *args, **kwargs):
+                super(DescriptiveUndefined, self).__init__(*args, **kwargs)
+                # When an undefined field is encountered, return the name
+                # of the undefined field in the exception message
+                # (StrictUndefined refers to the explicitly set exception
+                # message as the 'hint')
+                self._undefined_hint = self._undefined_name
+
+        # Ensure messages can be rendered
+        for msg in collected_messages:
+            env = sandbox.ImmutableSandboxedEnvironment(undefined=DescriptiveUndefined)
+            try:
+                env.from_string(msg).render(JobNotificationMixin.context_stub())
+            except TemplateSyntaxError as exc:
+                error_list.append(_("Unable to render message '{}': {}".format(msg, exc.message)))
+            except UndefinedError as exc:
+                error_list.append(_("Field '{}' unavailable".format(exc.message)))
+            except SecurityError as exc:
+                error_list.append(_("Security error due to field '{}'".format(exc.message)))
+
+        # Ensure that if a webhook body was provided, that it can be rendered as a dictionary
+        notification_type = ''
+        if self.instance:
+            notification_type = getattr(self.instance, 'notification_type', '')
+        else:
+            notification_type = self.initial_data.get('notification_type', '')
+
+        if notification_type == 'webhook':
+            for event in messages:
+                if not messages[event]:
+                    continue
+                body = messages[event].get('body', {})
+                if body:
+                    try:
+                        potential_body = json.loads(body)
+                        if not isinstance(potential_body, dict):
+                            error_list.append(_("Webhook body for '{}' should be a json dictionary. Found type '{}'."
+                                                .format(event, type(potential_body).__name__)))
+                    except json.JSONDecodeError as exc:
+                        error_list.append(_("Webhook body for '{}' is not a valid json dictionary ({}).".format(event, exc)))
+
+        if error_list:
+            raise serializers.ValidationError(error_list)
+
+        return messages
+
     def validate(self, attrs):
         from awx.api.views import NotificationTemplateDetail
 
@@ -4193,6 +4378,7 @@ class NotificationTemplateSerializer(BaseSerializer):
         notification_class = NotificationTemplate.CLASS_FOR_NOTIFICATION_TYPE[notification_type]
         missing_fields = []
         incorrect_type_fields = []
+        password_fields_to_forward = []
         error_list = []
         if 'notification_configuration' not in attrs:
             return attrs
@@ -4217,7 +4403,9 @@ class NotificationTemplateSerializer(BaseSerializer):
                 error_list.append(_("No values specified for field '{}'").format(field))
                 continue
             if field_type == "password" and field_val == "$encrypted$" and object_actual is not None:
-                attrs['notification_configuration'][field] = object_actual.notification_configuration[field]
+                password_fields_to_forward.append(field)
+            if field == "http_method" and field_val.lower() not in ['put', 'post']:
+                error_list.append(_("HTTP method must be either 'POST' or 'PUT'."))
         if missing_fields:
             error_list.append(_("Missing required fields for Notification Configuration: {}.").format(missing_fields))
         if incorrect_type_fields:
@@ -4226,15 +4414,31 @@ class NotificationTemplateSerializer(BaseSerializer):
                                                                                                     type_field_error[1]))
         if error_list:
             raise serializers.ValidationError(error_list)
+
+        # Only pull the existing encrypted passwords from the existing objects
+        # to assign to the attribute and forward on the call stack IF AND ONLY IF
+        # we know an error will not be raised in the validation phase.
+        # Otherwise, the encrypted password will be exposed.
+        for field in password_fields_to_forward:
+            attrs['notification_configuration'][field] = object_actual.notification_configuration[field]
         return super(NotificationTemplateSerializer, self).validate(attrs)
 
 
 class NotificationSerializer(BaseSerializer):
 
+    body = serializers.SerializerMethodField(
+        help_text=_('Notification body')
+    )
+
     class Meta:
         model = Notification
         fields = ('*', '-name', '-description', 'notification_template', 'error', 'status', 'notifications_sent',
-                  'notification_type', 'recipients', 'subject')
+                  'notification_type', 'recipients', 'subject', 'body')
+
+    def get_body(self, obj):
+        if obj.notification_type == 'webhook' and 'body' in obj.body:
+            return obj.body['body']
+        return obj.body
 
     def get_related(self, obj):
         res = super(NotificationSerializer, self).get_related(obj)
@@ -4243,6 +4447,15 @@ class NotificationSerializer(BaseSerializer):
         ))
         return res
 
+    def to_representation(self, obj):
+        ret = super(NotificationSerializer, self).to_representation(obj)
+
+        if obj.notification_type == 'webhook':
+            ret.pop('subject')
+        if obj.notification_type not in ('email', 'webhook', 'pagerduty'):
+            ret.pop('body')
+        return ret
+
 
 class LabelSerializer(BaseSerializer):
 
@@ -4555,7 +4768,8 @@ class ActivityStreamSerializer(BaseSerializer):
             ('o_auth2_access_token', ('id', 'user_id', 'description', 'application_id', 'scope')),
             ('o_auth2_application', ('id', 'name', 'description')),
             ('credential_type', ('id', 'name', 'description', 'kind', 'managed_by_tower')),
-            ('ad_hoc_command', ('id', 'name', 'status', 'limit'))
+            ('ad_hoc_command', ('id', 'name', 'status', 'limit')),
+            ('workflow_approval', ('id', 'name', 'unified_job_id')),
         ]
         return field_list
 
@@ -4621,37 +4835,37 @@ class ActivityStreamSerializer(BaseSerializer):
             return ""
 
     def get_related(self, obj):
-        rel = {}
+        data = {}
         if obj.actor is not None:
-            rel['actor'] = self.reverse('api:user_detail', kwargs={'pk': obj.actor.pk})
+            data['actor'] = self.reverse('api:user_detail', kwargs={'pk': obj.actor.pk})
         for fk, __ in self._local_summarizable_fk_fields:
             if not hasattr(obj, fk):
                 continue
-            m2m_list = self._get_rel(obj, fk)
+            m2m_list = self._get_related_objects(obj, fk)
             if m2m_list:
-                rel[fk] = []
+                data[fk] = []
                 id_list = []
-                for thisItem in m2m_list:
-                    if getattr(thisItem, 'id', None) in id_list:
+                for item in m2m_list:
+                    if getattr(item, 'id', None) in id_list:
                         continue
-                    id_list.append(getattr(thisItem, 'id', None))
-                    if hasattr(thisItem, 'get_absolute_url'):
-                        rel_url = thisItem.get_absolute_url(self.context.get('request'))
+                    id_list.append(getattr(item, 'id', None))
+                    if hasattr(item, 'get_absolute_url'):
+                        url = item.get_absolute_url(self.context.get('request'))
                     else:
                         view_name = fk + '_detail'
-                        rel_url = self.reverse('api:' + view_name, kwargs={'pk': thisItem.id})
-                    rel[fk].append(rel_url)
+                        url = self.reverse('api:' + view_name, kwargs={'pk': item.id})
+                    data[fk].append(url)
 
                     if fk == 'schedule':
-                        rel['unified_job_template'] = thisItem.unified_job_template.get_absolute_url(self.context.get('request'))
+                        data['unified_job_template'] = item.unified_job_template.get_absolute_url(self.context.get('request'))
         if obj.setting and obj.setting.get('category', None):
-            rel['setting'] = self.reverse(
+            data['setting'] = self.reverse(
                 'api:setting_singleton_detail',
                 kwargs={'category_slug': obj.setting['category']}
             )
-        return rel
+        return data
 
-    def _get_rel(self, obj, fk):
+    def _get_related_objects(self, obj, fk):
         related_model = ActivityStream._meta.get_field(fk).related_model
         related_manager = getattr(obj, fk)
         if issubclass(related_model, PolymorphicModel) and hasattr(obj, '_prefetched_objects_cache'):
@@ -4661,43 +4875,36 @@ class ActivityStreamSerializer(BaseSerializer):
                 obj._prefetched_objects_cache[related_manager.prefetch_cache_name] = list(related_manager.all())
         return related_manager.all()
 
+    def _summarize_parent_ujt(self, obj, fk, summary_fields):
+        summary_keys = {'job': 'job_template',
+                        'workflow_job_template_node': 'workflow_job_template',
+                        'workflow_approval_template': 'workflow_job_template',
+                        'workflow_approval': 'workflow_job',
+                        'schedule': 'unified_job_template'}
+        if fk not in summary_keys:
+            return
+        related_obj = getattr(obj, summary_keys[fk], None)
+        item = {}
+        fields = SUMMARIZABLE_FK_FIELDS[summary_keys[fk]]
+        if related_obj is not None:
+            summary_fields[get_type_for_model(related_obj)] = []
+            for field in fields:
+                fval = getattr(related_obj, field, None)
+                if fval is not None:
+                    item[field] = fval
+            summary_fields[get_type_for_model(related_obj)].append(item)
+
     def get_summary_fields(self, obj):
         summary_fields = OrderedDict()
         for fk, related_fields in self._local_summarizable_fk_fields:
             try:
                 if not hasattr(obj, fk):
                     continue
-                m2m_list = self._get_rel(obj, fk)
+                m2m_list = self._get_related_objects(obj, fk)
                 if m2m_list:
                     summary_fields[fk] = []
                     for thisItem in m2m_list:
-                        if fk == 'job':
-                            summary_fields['job_template'] = []
-                            job_template_item = {}
-                            job_template_fields = SUMMARIZABLE_FK_FIELDS['job_template']
-                            job_template = getattr(thisItem, 'job_template', None)
-                            if job_template is not None:
-                                for field in job_template_fields:
-                                    fval = getattr(job_template, field, None)
-                                    if fval is not None:
-                                        job_template_item[field] = fval
-                                summary_fields['job_template'].append(job_template_item)
-                        if fk == 'workflow_job_template_node':
-                            summary_fields['workflow_job_template'] = []
-                            workflow_job_template_item = {}
-                            workflow_job_template_fields = SUMMARIZABLE_FK_FIELDS['workflow_job_template']
-                            workflow_job_template = getattr(thisItem, 'workflow_job_template', None)
-                            if workflow_job_template is not None:
-                                for field in workflow_job_template_fields:
-                                    fval = getattr(workflow_job_template, field, None)
-                                    if fval is not None:
-                                        workflow_job_template_item[field] = fval
-                                summary_fields['workflow_job_template'].append(workflow_job_template_item)
-                        if fk == 'schedule':
-                            unified_job_template = getattr(thisItem, 'unified_job_template', None)
-                            if unified_job_template is not None:
-                                summary_fields[get_type_for_model(unified_job_template)] = {'id': unified_job_template.id,
-                                                                                            'name': unified_job_template.name}
+                        self._summarize_parent_ujt(thisItem, fk, summary_fields)
                         thisItemDict = {}
                         for field in related_fields:
                             fval = getattr(thisItem, field, None)

awx/api/urls/inventory_source.py

@@ -13,8 +13,8 @@ from awx.api.views import (
     InventorySourceCredentialsList,
     InventorySourceGroupsList,
     InventorySourceHostsList,
-    InventorySourceNotificationTemplatesAnyList,
     InventorySourceNotificationTemplatesErrorList,
+    InventorySourceNotificationTemplatesStartedList,
     InventorySourceNotificationTemplatesSuccessList,
 )
 
@@ -29,8 +29,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/credentials/$', InventorySourceCredentialsList.as_view(), name='inventory_source_credentials_list'),
     url(r'^(?P<pk>[0-9]+)/groups/$', InventorySourceGroupsList.as_view(), name='inventory_source_groups_list'),
     url(r'^(?P<pk>[0-9]+)/hosts/$', InventorySourceHostsList.as_view(), name='inventory_source_hosts_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', InventorySourceNotificationTemplatesAnyList.as_view(),
-        name='inventory_source_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', InventorySourceNotificationTemplatesStartedList.as_view(),
+        name='inventory_source_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', InventorySourceNotificationTemplatesErrorList.as_view(),
         name='inventory_source_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', InventorySourceNotificationTemplatesSuccessList.as_view(),

awx/api/urls/job_template.py

@@ -13,8 +13,8 @@ from awx.api.views import (
     JobTemplateSchedulesList,
     JobTemplateSurveySpec,
     JobTemplateActivityStreamList,
-    JobTemplateNotificationTemplatesAnyList,
     JobTemplateNotificationTemplatesErrorList,
+    JobTemplateNotificationTemplatesStartedList,
     JobTemplateNotificationTemplatesSuccessList,
     JobTemplateInstanceGroupsList,
     JobTemplateAccessList,
@@ -34,8 +34,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/schedules/$', JobTemplateSchedulesList.as_view(), name='job_template_schedules_list'),
     url(r'^(?P<pk>[0-9]+)/survey_spec/$', JobTemplateSurveySpec.as_view(), name='job_template_survey_spec'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', JobTemplateActivityStreamList.as_view(), name='job_template_activity_stream_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', JobTemplateNotificationTemplatesAnyList.as_view(),
-        name='job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', JobTemplateNotificationTemplatesStartedList.as_view(),
+        name='job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', JobTemplateNotificationTemplatesErrorList.as_view(),
         name='job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', JobTemplateNotificationTemplatesSuccessList.as_view(),

awx/api/urls/organization.py

@@ -15,8 +15,8 @@ from awx.api.views import (
     OrganizationCredentialList,
     OrganizationActivityStreamList,
     OrganizationNotificationTemplatesList,
-    OrganizationNotificationTemplatesAnyList,
     OrganizationNotificationTemplatesErrorList,
+    OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
     OrganizationInstanceGroupsList,
     OrganizationObjectRolesList,
@@ -25,7 +25,7 @@ from awx.api.views import (
 )
 
 
-urls = [ 
+urls = [
     url(r'^$', OrganizationList.as_view(), name='organization_list'),
     url(r'^(?P<pk>[0-9]+)/$', OrganizationDetail.as_view(), name='organization_detail'),
     url(r'^(?P<pk>[0-9]+)/users/$', OrganizationUsersList.as_view(), name='organization_users_list'),
@@ -37,8 +37,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/credentials/$', OrganizationCredentialList.as_view(), name='organization_credential_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', OrganizationActivityStreamList.as_view(), name='organization_activity_stream_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates/$', OrganizationNotificationTemplatesList.as_view(), name='organization_notification_templates_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', OrganizationNotificationTemplatesAnyList.as_view(),
-        name='organization_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', OrganizationNotificationTemplatesStartedList.as_view(),
+        name='organization_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', OrganizationNotificationTemplatesErrorList.as_view(),
         name='organization_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', OrganizationNotificationTemplatesSuccessList.as_view(),

awx/api/urls/project.py

@@ -14,8 +14,8 @@ from awx.api.views import (
     ProjectUpdatesList,
     ProjectActivityStreamList,
     ProjectSchedulesList,
-    ProjectNotificationTemplatesAnyList,
     ProjectNotificationTemplatesErrorList,
+    ProjectNotificationTemplatesStartedList,
     ProjectNotificationTemplatesSuccessList,
     ProjectObjectRolesList,
     ProjectAccessList,
@@ -34,10 +34,11 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/project_updates/$', ProjectUpdatesList.as_view(), name='project_updates_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', ProjectActivityStreamList.as_view(), name='project_activity_stream_list'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', ProjectSchedulesList.as_view(), name='project_schedules_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', ProjectNotificationTemplatesAnyList.as_view(), name='project_notification_templates_any_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', ProjectNotificationTemplatesErrorList.as_view(), name='project_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', ProjectNotificationTemplatesSuccessList.as_view(),
         name='project_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', ProjectNotificationTemplatesStartedList.as_view(),
+        name='project_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', ProjectObjectRolesList.as_view(), name='project_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', ProjectAccessList.as_view(), name='project_access_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', ProjectCopy.as_view(), name='project_copy'),

awx/api/urls/system_job_template.py

@@ -9,8 +9,8 @@ from awx.api.views import (
     SystemJobTemplateLaunch,
     SystemJobTemplateJobsList,
     SystemJobTemplateSchedulesList,
-    SystemJobTemplateNotificationTemplatesAnyList,
     SystemJobTemplateNotificationTemplatesErrorList,
+    SystemJobTemplateNotificationTemplatesStartedList,
     SystemJobTemplateNotificationTemplatesSuccessList,
 )
 
@@ -21,8 +21,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/launch/$', SystemJobTemplateLaunch.as_view(), name='system_job_template_launch'),
     url(r'^(?P<pk>[0-9]+)/jobs/$', SystemJobTemplateJobsList.as_view(), name='system_job_template_jobs_list'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', SystemJobTemplateSchedulesList.as_view(), name='system_job_template_schedules_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', SystemJobTemplateNotificationTemplatesAnyList.as_view(),
-        name='system_job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', SystemJobTemplateNotificationTemplatesStartedList.as_view(),
+        name='system_job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', SystemJobTemplateNotificationTemplatesErrorList.as_view(),
         name='system_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', SystemJobTemplateNotificationTemplatesSuccessList.as_view(),

awx/api/urls/urls.py

@@ -71,6 +71,8 @@ from .instance import urls as instance_urls
 from .instance_group import urls as instance_group_urls
 from .oauth2 import urls as oauth2_urls
 from .oauth2_root import urls as oauth2_root_urls
+from .workflow_approval_template import urls as workflow_approval_template_urls
+from .workflow_approval import urls as workflow_approval_urls
 
 
 v2_urls = [
@@ -131,8 +133,11 @@ v2_urls = [
     url(r'^unified_job_templates/$', UnifiedJobTemplateList.as_view(), name='unified_job_template_list'),
     url(r'^unified_jobs/$', UnifiedJobList.as_view(), name='unified_job_list'),
     url(r'^activity_stream/', include(activity_stream_urls)),
+    url(r'^workflow_approval_templates/', include(workflow_approval_template_urls)),
+    url(r'^workflow_approvals/', include(workflow_approval_urls)),
 ]
 
+
 app_name = 'api'
 urlpatterns = [
     url(r'^$', ApiRootView.as_view(), name='api_root_view'),

awx/api/urls/workflow_approval.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    WorkflowApprovalList,
+    WorkflowApprovalDetail,
+    WorkflowApprovalApprove,
+    WorkflowApprovalDeny,
+)
+
+
+urls = [
+    url(r'^$', WorkflowApprovalList.as_view(), name='workflow_approval_list'),
+    url(r'^(?P<pk>[0-9]+)/$', WorkflowApprovalDetail.as_view(), name='workflow_approval_detail'),
+    url(r'^(?P<pk>[0-9]+)/approve/$', WorkflowApprovalApprove.as_view(), name='workflow_approval_approve'),
+    url(r'^(?P<pk>[0-9]+)/deny/$', WorkflowApprovalDeny.as_view(), name='workflow_approval_deny'),
+]
+
+__all__ = ['urls']

awx/api/urls/workflow_approval_template.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    WorkflowApprovalTemplateDetail,
+    WorkflowApprovalTemplateJobsList,
+)
+
+
+urls = [
+    url(r'^(?P<pk>[0-9]+)/$', WorkflowApprovalTemplateDetail.as_view(), name='workflow_approval_template_detail'),
+    url(r'^(?P<pk>[0-9]+)/approvals/$', WorkflowApprovalTemplateJobsList.as_view(), name='workflow_approval_template_jobs_list'),
+]
+
+__all__ = ['urls']

awx/api/urls/workflow_job_template.py

@@ -13,8 +13,8 @@ from awx.api.views import (
     WorkflowJobTemplateSurveySpec,
     WorkflowJobTemplateWorkflowNodesList,
     WorkflowJobTemplateActivityStreamList,
-    WorkflowJobTemplateNotificationTemplatesAnyList,
     WorkflowJobTemplateNotificationTemplatesErrorList,
+    WorkflowJobTemplateNotificationTemplatesStartedList,
     WorkflowJobTemplateNotificationTemplatesSuccessList,
     WorkflowJobTemplateAccessList,
     WorkflowJobTemplateObjectRolesList,
@@ -32,8 +32,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/survey_spec/$', WorkflowJobTemplateSurveySpec.as_view(), name='workflow_job_template_survey_spec'),
     url(r'^(?P<pk>[0-9]+)/workflow_nodes/$', WorkflowJobTemplateWorkflowNodesList.as_view(), name='workflow_job_template_workflow_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', WorkflowJobTemplateActivityStreamList.as_view(), name='workflow_job_template_activity_stream_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', WorkflowJobTemplateNotificationTemplatesAnyList.as_view(),
-        name='workflow_job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', WorkflowJobTemplateNotificationTemplatesStartedList.as_view(),
+        name='workflow_job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', WorkflowJobTemplateNotificationTemplatesErrorList.as_view(),
         name='workflow_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', WorkflowJobTemplateNotificationTemplatesSuccessList.as_view(),

awx/api/urls/workflow_job_template_node.py

@@ -10,6 +10,7 @@ from awx.api.views import (
     WorkflowJobTemplateNodeFailureNodesList,
     WorkflowJobTemplateNodeAlwaysNodesList,
     WorkflowJobTemplateNodeCredentialsList,
+    WorkflowJobTemplateNodeCreateApproval,
 )
 
 
@@ -20,6 +21,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/failure_nodes/$', WorkflowJobTemplateNodeFailureNodesList.as_view(), name='workflow_job_template_node_failure_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/always_nodes/$', WorkflowJobTemplateNodeAlwaysNodesList.as_view(), name='workflow_job_template_node_always_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/credentials/$', WorkflowJobTemplateNodeCredentialsList.as_view(), name='workflow_job_template_node_credentials_list'),
+    url(r'^(?P<pk>[0-9]+)/create_approval_template/$', WorkflowJobTemplateNodeCreateApproval.as_view(), name='workflow_job_template_node_create_approval'),
 ]
 
 __all__ = ['urls']

awx/api/views/__init__.py

@@ -91,7 +91,8 @@ from awx.main.redact import UriCleaner
 from awx.api.permissions import (
     JobTemplateCallbackPermission, TaskPermission, ProjectUpdatePermission,
     InventoryInventorySourcesUpdatePermission, UserPermission,
-    InstanceGroupTowerPermission, VariableDataPermission
+    InstanceGroupTowerPermission, VariableDataPermission,
+    WorkflowApprovalPermission
 )
 from awx.api import renderers
 from awx.api import serializers
@@ -116,6 +117,7 @@ from awx.api.views.organization import ( # noqa
     OrganizationNotificationTemplatesList,
     OrganizationNotificationTemplatesAnyList,
     OrganizationNotificationTemplatesErrorList,
+    OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
     OrganizationInstanceGroupsList,
     OrganizationAccessList,
@@ -747,22 +749,20 @@ class ProjectNotificationTemplatesAnyList(SubListCreateAttachDetachAPIView):
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.Project
-    relationship = 'notification_templates_any'
 
 
-class ProjectNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class ProjectNotificationTemplatesStartedList(ProjectNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class ProjectNotificationTemplatesErrorList(ProjectNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.Project
     relationship = 'notification_templates_error'
 
 
-class ProjectNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class ProjectNotificationTemplatesSuccessList(ProjectNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.Project
     relationship = 'notification_templates_success'
 
 
@@ -840,8 +840,6 @@ class SystemJobEventsList(SubListAPIView):
         return super(SystemJobEventsList, self).finalize_response(request, response, *args, **kwargs)
 
 
-
-
 class ProjectUpdateCancel(RetrieveAPIView):
 
     model = models.ProjectUpdate
@@ -2102,7 +2100,6 @@ class InventorySourceNotificationTemplatesAnyList(SubListCreateAttachDetachAPIVi
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.InventorySource
-    relationship = 'notification_templates_any'
 
     def post(self, request, *args, **kwargs):
         parent = self.get_parent_object()
@@ -2113,6 +2110,11 @@ class InventorySourceNotificationTemplatesAnyList(SubListCreateAttachDetachAPIVi
         return super(InventorySourceNotificationTemplatesAnyList, self).post(request, *args, **kwargs)
 
 
+class InventorySourceNotificationTemplatesStartedList(InventorySourceNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
 class InventorySourceNotificationTemplatesErrorList(InventorySourceNotificationTemplatesAnyList):
 
     relationship = 'notification_templates_error'
@@ -2626,22 +2628,20 @@ class JobTemplateNotificationTemplatesAnyList(SubListCreateAttachDetachAPIView):
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.JobTemplate
-    relationship = 'notification_templates_any'
 
 
-class JobTemplateNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class JobTemplateNotificationTemplatesStartedList(JobTemplateNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class JobTemplateNotificationTemplatesErrorList(JobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.JobTemplate
     relationship = 'notification_templates_error'
 
 
-class JobTemplateNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class JobTemplateNotificationTemplatesSuccessList(JobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.JobTemplate
     relationship = 'notification_templates_success'
 
 
@@ -2996,7 +2996,7 @@ class WorkflowJobTemplateNodeChildrenBaseList(EnforceParentRelationshipMixin, Su
         relationships = ['success_nodes', 'failure_nodes', 'always_nodes']
         relationships.remove(self.relationship)
         qs = functools.reduce(lambda x, y: (x | y),
-                              (Q(**{'{}__in'.format(rel): [sub.id]}) for rel in relationships))
+                              (Q(**{'{}__in'.format(r): [sub.id]}) for r in relationships))
 
         if models.WorkflowJobTemplateNode.objects.filter(Q(pk=parent.id) & qs).exists():
             return {"Error": _("Relationship not allowed.")}
@@ -3012,6 +3012,34 @@ class WorkflowJobTemplateNodeChildrenBaseList(EnforceParentRelationshipMixin, Su
         return None
 
 
+class WorkflowJobTemplateNodeCreateApproval(RetrieveAPIView):
+
+    model = models.WorkflowJobTemplateNode
+    serializer_class = serializers.WorkflowJobTemplateNodeCreateApprovalSerializer
+    permission_classes = []
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        serializer = self.get_serializer(instance=obj, data=request.data)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+        approval_template = obj.create_approval_template(**serializer.validated_data)
+        data = serializers.WorkflowApprovalTemplateSerializer(
+            approval_template,
+            context=self.get_serializer_context()
+        ).data
+        return Response(data, status=status.HTTP_200_OK)
+
+    def check_permissions(self, request):
+        obj = self.get_object().workflow_job_template
+        if request.method == 'POST':
+            if not request.user.can_access(models.WorkflowJobTemplate, 'change', obj, request.data):
+                self.permission_denied(request)
+        else:
+            if not request.user.can_access(models.WorkflowJobTemplate, 'read', obj):
+                self.permission_denied(request)
+
+
 class WorkflowJobTemplateNodeSuccessNodesList(WorkflowJobTemplateNodeChildrenBaseList):
     relationship = 'success_nodes'
 
@@ -3234,22 +3262,20 @@ class WorkflowJobTemplateNotificationTemplatesAnyList(SubListCreateAttachDetachA
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.WorkflowJobTemplate
-    relationship = 'notification_templates_any'
 
 
-class WorkflowJobTemplateNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class WorkflowJobTemplateNotificationTemplatesStartedList(WorkflowJobTemplateNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class WorkflowJobTemplateNotificationTemplatesErrorList(WorkflowJobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.WorkflowJobTemplate
     relationship = 'notification_templates_error'
 
 
-class WorkflowJobTemplateNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class WorkflowJobTemplateNotificationTemplatesSuccessList(WorkflowJobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.WorkflowJobTemplate
     relationship = 'notification_templates_success'
 
 
@@ -3288,7 +3314,7 @@ class WorkflowJobTemplateActivityStreamList(SubListAPIView):
                          Q(workflow_job_template_node__workflow_job_template=parent)).distinct()
 
 
-class WorkflowJobList(ListCreateAPIView):
+class WorkflowJobList(ListAPIView):
 
     model = models.WorkflowJob
     serializer_class = serializers.WorkflowJobListSerializer
@@ -3411,22 +3437,20 @@ class SystemJobTemplateNotificationTemplatesAnyList(SubListCreateAttachDetachAPI
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.SystemJobTemplate
-    relationship = 'notification_templates_any'
 
 
-class SystemJobTemplateNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class SystemJobTemplateNotificationTemplatesStartedList(SystemJobTemplateNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class SystemJobTemplateNotificationTemplatesErrorList(SystemJobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.SystemJobTemplate
     relationship = 'notification_templates_error'
 
 
-class SystemJobTemplateNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class SystemJobTemplateNotificationTemplatesSuccessList(SystemJobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.SystemJobTemplate
     relationship = 'notification_templates_success'
 
 
@@ -3978,7 +4002,7 @@ class AdHocCommandNotificationsList(SubListAPIView):
     search_fields = ('subject', 'notification_type', 'body',)
 
 
-class SystemJobList(ListCreateAPIView):
+class SystemJobList(ListAPIView):
 
     model = models.SystemJob
     serializer_class = serializers.SystemJobListSerializer
@@ -4408,3 +4432,63 @@ for attr, value in list(locals().items()):
         name = camelcase_to_underscore(attr)
         view = value.as_view()
         setattr(this_module, name, view)
+
+
+class WorkflowApprovalTemplateDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
+
+    model = models.WorkflowApprovalTemplate
+    serializer_class = serializers.WorkflowApprovalTemplateSerializer
+
+
+class WorkflowApprovalTemplateJobsList(SubListAPIView):
+
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalListSerializer
+    parent_model = models.WorkflowApprovalTemplate
+    relationship = 'approvals'
+    parent_key = 'workflow_approval_template'
+
+
+class WorkflowApprovalList(ListCreateAPIView):
+
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalListSerializer
+
+    def get(self, request, *args, **kwargs):
+        return super(WorkflowApprovalList, self).get(request, *args, **kwargs)
+
+
+class WorkflowApprovalDetail(UnifiedJobDeletionMixin, RetrieveDestroyAPIView):
+
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalSerializer
+
+
+class WorkflowApprovalApprove(RetrieveAPIView):
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalViewSerializer
+    permission_classes = (WorkflowApprovalPermission,)
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        if not request.user.can_access(models.WorkflowApproval, 'approve_or_deny', obj):
+            raise PermissionDenied(detail=_("User does not have permission to approve or deny this workflow."))
+        if obj.status != 'pending':
+            return Response({"error": _("This workflow step has already been approved or denied.")}, status=status.HTTP_400_BAD_REQUEST)
+        obj.approve(request)
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+class WorkflowApprovalDeny(RetrieveAPIView):
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalViewSerializer
+    permission_classes = (WorkflowApprovalPermission,)
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        if not request.user.can_access(models.WorkflowApproval, 'approve_or_deny', obj):
+            raise PermissionDenied(detail=_("User does not have permission to approve or deny this workflow."))
+        if obj.status != 'pending':
+            return Response({"error": _("This workflow step has already been approved or denied.")}, status=status.HTTP_400_BAD_REQUEST)
+        obj.deny(request)
+        return Response(status=status.HTTP_204_NO_CONTENT)

awx/api/views/metrics.py

@@ -31,9 +31,10 @@ class MetricsView(APIView):
     swagger_topic = 'Metrics'
 
     renderer_classes = [renderers.PlainTextRenderer,
+                        renderers.PrometheusJSONRenderer,
                         renderers.BrowsableAPIRenderer,]
 
-    def get(self, request, format='txt'):
+    def get(self, request):
         ''' Show Metrics Details '''
         if (request.user.is_superuser or request.user.is_system_auditor):
             return Response(metrics().decode('UTF-8'))

awx/api/views/organization.py

@@ -178,22 +178,20 @@ class OrganizationNotificationTemplatesAnyList(SubListCreateAttachDetachAPIView)
     model = NotificationTemplate
     serializer_class = NotificationTemplateSerializer
     parent_model = Organization
-    relationship = 'notification_templates_any'
 
 
-class OrganizationNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class OrganizationNotificationTemplatesStartedList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class OrganizationNotificationTemplatesErrorList(OrganizationNotificationTemplatesAnyList):
 
-    model = NotificationTemplate
-    serializer_class = NotificationTemplateSerializer
-    parent_model = Organization
     relationship = 'notification_templates_error'
 
 
-class OrganizationNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class OrganizationNotificationTemplatesSuccessList(OrganizationNotificationTemplatesAnyList):
 
-    model = NotificationTemplate
-    serializer_class = NotificationTemplateSerializer
-    parent_model = Organization
     relationship = 'notification_templates_success'
 
 

awx/api/views/root.py

@@ -124,6 +124,7 @@ class ApiVersionRootView(APIView):
         data['activity_stream'] = reverse('api:activity_stream_list', request=request)
         data['workflow_job_templates'] = reverse('api:workflow_job_template_list', request=request)
         data['workflow_jobs'] = reverse('api:workflow_job_list', request=request)
+        data['workflow_approvals'] = reverse('api:workflow_approval_list', request=request)
         data['workflow_job_template_nodes'] = reverse('api:workflow_job_template_node_list', request=request)
         data['workflow_job_nodes'] = reverse('api:workflow_job_node_list', request=request)
         return Response(data)

awx/conf/migrations/0001_initial.py

@@ -21,7 +21,8 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('key', models.CharField(max_length=255)),
                 ('value', jsonfield.fields.JSONField(null=True)),
-                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('user', models.ForeignKey(related_name='settings', default=None, editable=False,
+                                           to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True)),
             ],
             options={
                 'abstract': False,

awx/conf/tests/functional/test_api.py

@@ -8,6 +8,7 @@ from awx.main.utils.encryption import decrypt_field
 from awx.conf import fields
 from awx.conf.registry import settings_registry
 from awx.conf.models import Setting
+from awx.sso import fields as sso_fields
 
 
 @pytest.fixture
@@ -137,7 +138,7 @@ def test_setting_signleton_retrieve_hierachy(api_request, dummy_setting):
 
 
 @pytest.mark.django_db
-def test_setting_signleton_retrieve_readonly(api_request, dummy_setting):
+def test_setting_singleton_retrieve_readonly(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.IntegerField,
@@ -183,6 +184,30 @@ def test_setting_singleton_update(api_request, dummy_setting):
         assert response.data['FOO_BAR'] == 4
 
 
+@pytest.mark.django_db
+def test_setting_singleton_update_hybriddictfield_with_forbidden(api_request, dummy_setting):
+    # Some HybridDictField subclasses have a child of _Forbidden,
+    # indicating that only the defined fields can be filled in.  Make
+    # sure that the _Forbidden validator doesn't get used for the
+    # fields.  See also https://github.com/ansible/awx/issues/4099.
+    with dummy_setting(
+        'FOO_BAR',
+        field_class=sso_fields.SAMLOrgAttrField,
+        category='FooBar',
+        category_slug='foobar',
+    ), mock.patch('awx.conf.views.handle_setting_changes'):
+        api_request(
+            'patch',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}),
+            data={'FOO_BAR': {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}}
+        )
+        response = api_request(
+            'get',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'})
+        )
+        assert response.data['FOO_BAR'] == {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}
+
+
 @pytest.mark.django_db
 def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy_setting):
     with dummy_setting(
@@ -206,7 +231,7 @@ def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy
 
 
 @pytest.mark.django_db
-def test_setting_singleton_update_dont_change_encripted_mark(api_request, dummy_setting):
+def test_setting_singleton_update_dont_change_encrypted_mark(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.CharField,

awx/main/access.py

@@ -37,6 +37,7 @@ from awx.main.models import (
     ProjectUpdateEvent, Role, Schedule, SystemJob, SystemJobEvent,
     SystemJobTemplate, Team, UnifiedJob, UnifiedJobTemplate, WorkflowJob,
     WorkflowJobNode, WorkflowJobTemplate, WorkflowJobTemplateNode,
+    WorkflowApproval, WorkflowApprovalTemplate,
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR
 )
 from awx.main.models.mixins import ResourceMixin
@@ -538,7 +539,7 @@ class InstanceGroupAccess(BaseAccess):
 
     def filtered_queryset(self):
         return InstanceGroup.objects.filter(
-            organization__in=Organization.accessible_pk_qs(self.user, 'admin_role'))
+            organization__in=Organization.accessible_pk_qs(self.user, 'admin_role')).distinct()
 
     def can_add(self, data):
         return self.user.is_superuser
@@ -833,10 +834,6 @@ class InventoryAccess(BaseAccess):
     def filtered_queryset(self, allowed=None, ad_hoc=None):
         return self.model.accessible_objects(self.user, 'read_role')
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.read_role
-
     @check_superuser
     def can_use(self, obj):
         return self.user in obj.use_role
@@ -906,9 +903,6 @@ class HostAccess(BaseAccess):
     def filtered_queryset(self):
         return self.model.objects.filter(inventory__in=Inventory.accessible_pk_qs(self.user, 'read_role'))
 
-    def can_read(self, obj):
-        return obj and self.user in obj.inventory.read_role
-
     def can_add(self, data):
         if not data:  # So the browseable API will work
             return Inventory.accessible_objects(self.user, 'admin_role').exists()
@@ -970,9 +964,6 @@ class GroupAccess(BaseAccess):
     def filtered_queryset(self):
         return Group.objects.filter(inventory__in=Inventory.accessible_pk_qs(self.user, 'read_role'))
 
-    def can_read(self, obj):
-        return obj and self.user in obj.inventory.read_role
-
     def can_add(self, data):
         if not data or 'inventory' not in data:
             return False
@@ -1016,12 +1007,6 @@ class InventorySourceAccess(NotificationAttachMixin, BaseAccess):
     def filtered_queryset(self):
         return self.model.objects.filter(inventory__in=Inventory.accessible_pk_qs(self.user, 'read_role'))
 
-    def can_read(self, obj):
-        if obj and obj.inventory:
-            return self.user.can_access(Inventory, 'read', obj.inventory)
-        else:
-            return False
-
     def can_add(self, data):
         if not data or 'inventory' not in data:
             return Organization.accessible_objects(self.user, 'admin_role').exists()
@@ -1114,9 +1099,6 @@ class CredentialTypeAccess(BaseAccess):
     model = CredentialType
     prefetch_related = ('created_by', 'modified_by',)
 
-    def can_read(self, obj):
-        return True
-
     def can_use(self, obj):
         return True
 
@@ -1158,25 +1140,26 @@ class CredentialAccess(BaseAccess):
     def filtered_queryset(self):
         return self.model.accessible_objects(self.user, 'read_role')
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.read_role
-
     @check_superuser
     def can_add(self, data):
         if not data:  # So the browseable API will work
             return True
         if data and data.get('user', None):
             user_obj = get_object_from_data('user', User, data)
-            return bool(self.user == user_obj or UserAccess(self.user).can_admin(user_obj, None, check_setting=False))
+            if not bool(self.user == user_obj or UserAccess(self.user).can_admin(user_obj, None, check_setting=False)):
+                return False
         if data and data.get('team', None):
             team_obj = get_object_from_data('team', Team, data)
-            return check_user_access(self.user, Team, 'change', team_obj, None)
+            if not check_user_access(self.user, Team, 'change', team_obj, None):
+                return False
         if data and data.get('organization', None):
             organization_obj = get_object_from_data('organization', Organization, data)
-            return any([check_user_access(self.user, Organization, 'change', organization_obj, None),
-                        self.user in organization_obj.credential_admin_role])
-        return False
+            if not any([check_user_access(self.user, Organization, 'change', organization_obj, None),
+                        self.user in organization_obj.credential_admin_role]):
+                return False
+        if not any(data.get(key, None) for key in ('user', 'team', 'organization')):
+            return False  # you have to provide 1 owner field
+        return True
 
     @check_superuser
     def can_use(self, obj):
@@ -1219,10 +1202,6 @@ class CredentialInputSourceAccess(BaseAccess):
         return CredentialInputSource.objects.filter(
             target_credential__in=Credential.accessible_pk_qs(self.user, 'read_role'))
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.target_credential.read_role
-
     @check_superuser
     def can_add(self, data):
         return (
@@ -1265,7 +1244,7 @@ class TeamAccess(BaseAccess):
                 (self.user.admin_of_organizations.exists() or self.user.auditor_of_organizations.exists()):
             return self.model.objects.all()
         return self.model.objects.filter(
-            Q(organization=Organization.accessible_pk_qs(self.user, 'member_role')) |
+            Q(organization__in=Organization.accessible_pk_qs(self.user, 'member_role')) |
             Q(pk__in=self.model.accessible_pk_qs(self.user, 'read_role'))
         )
 
@@ -1971,10 +1950,6 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess):
     def filtered_queryset(self):
         return self.model.accessible_objects(self.user, 'read_role')
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.read_role
-
     @check_superuser
     def can_add(self, data):
         '''
@@ -2372,13 +2347,18 @@ class UnifiedJobTemplateAccess(BaseAccess):
         return self.model.objects.filter(
             Q(pk__in=self.model.accessible_pk_qs(self.user, 'read_role')) |
             Q(inventorysource__inventory__id__in=Inventory._accessible_pk_qs(
-                Inventory, self.user, 'read_role')))
+                Inventory, self.user, 'read_role'))
+        )
 
     def can_start(self, obj, validate_license=True):
         access_class = access_registry[obj.__class__]
         access_instance = access_class(self.user)
         return access_instance.can_start(obj, validate_license=validate_license)
 
+    def get_queryset(self):
+        return super(UnifiedJobTemplateAccess, self).get_queryset().filter(
+            workflowapprovaltemplate__isnull=True)
+
 
 class UnifiedJobAccess(BaseAccess):
     '''
@@ -2425,6 +2405,10 @@ class UnifiedJobAccess(BaseAccess):
         )
         return qs
 
+    def get_queryset(self):
+        return super(UnifiedJobAccess, self).get_queryset().filter(
+            workflowapproval__isnull=True)
+
 
 class ScheduleAccess(BaseAccess):
     '''
@@ -2486,14 +2470,6 @@ class NotificationTemplateAccess(BaseAccess):
             Q(organization__in=self.user.auditor_of_organizations)
         ).distinct()
 
-    def can_read(self, obj):
-        if self.user.is_superuser or self.user.is_system_auditor:
-            return True
-        if obj.organization is not None:
-            if self.user in obj.organization.notification_admin_role or self.user in obj.organization.auditor_role:
-                return True
-        return False
-
     @check_superuser
     def can_add(self, data):
         if not data:
@@ -2533,9 +2509,6 @@ class NotificationAccess(BaseAccess):
             Q(notification_template__organization__in=self.user.auditor_of_organizations)
         ).distinct()
 
-    def can_read(self, obj):
-        return self.user.can_access(NotificationTemplate, 'read', obj.notification_template)
-
     def can_delete(self, obj):
         return self.user.can_access(NotificationTemplate, 'delete', obj.notification_template)
 
@@ -2550,10 +2523,6 @@ class LabelAccess(BaseAccess):
     def filtered_queryset(self):
         return self.model.objects.all()
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.organization.read_role
-
     @check_superuser
     def can_add(self, data):
         if not data:  # So the browseable API will work
@@ -2711,15 +2680,6 @@ class RoleAccess(BaseAccess):
             result = result | super_qs
         return result
 
-    def can_read(self, obj):
-        if not obj:
-            return False
-        if self.user.is_superuser or self.user.is_system_auditor:
-            return True
-
-        return Role.filter_visible_roles(
-            self.user, Role.objects.filter(pk=obj.id)).exists()
-
     def can_add(self, obj, data):
         # Unsupported for now
         return False
@@ -2764,5 +2724,80 @@ class RoleAccess(BaseAccess):
         return False
 
 
+class WorkflowApprovalAccess(BaseAccess):
+    '''
+    A user can create a workflow approval if they are a superuser, an org admin
+    of the org connected to the workflow, or if they are assigned as admins to
+    the workflow.
+
+    A user can approve a workflow when they are:
+    - a superuser
+    - a workflow admin
+    - an organization admin
+    - any user who has explicitly been assigned the "approver" role
+
+    A user can see approvals if they have read access to the associated WorkflowJobTemplate.
+    '''
+
+    model = WorkflowApproval
+    prefetch_related = ('created_by', 'modified_by',)
+
+    def can_use(self, obj):
+        return True
+
+    def can_start(self, obj, validate_license=True):
+        return True
+
+    def filtered_queryset(self):
+        return self.model.objects.filter(
+            unified_job_node__workflow_job__unified_job_template__in=WorkflowJobTemplate.accessible_pk_qs(
+                self.user, 'read_role'))
+
+    def can_approve_or_deny(self, obj):
+        if (
+            (obj.workflow_job_template and self.user in obj.workflow_job_template.approval_role) or
+            self.user.is_superuser
+        ):
+            return True
+
+
+class WorkflowApprovalTemplateAccess(BaseAccess):
+    '''
+    A user can create a workflow approval if they are a superuser, an org admin
+    of the org connected to the workflow, or if they are assigned as admins to
+    the workflow.
+
+    A user can approve a workflow when they are:
+    - a superuser
+    - a workflow admin
+    - an organization admin
+    - any user who has explicitly been assigned the "approver" role at the workflow or organization level
+
+    A user can see approval templates if they have read access to the associated WorkflowJobTemplate.
+    '''
+
+    model = WorkflowApprovalTemplate
+    prefetch_related = ('created_by', 'modified_by',)
+
+    @check_superuser
+    def can_add(self, data):
+        if data is None:  # Hide direct creation in API browser
+            return False
+        else:
+            return (self.check_related('workflow_approval_template', UnifiedJobTemplate, role_field='admin_role'))
+
+    def can_start(self, obj, validate_license=False):
+        # for copying WFJTs that contain approval nodes
+        if self.user.is_superuser:
+            return True
+
+        return self.user in obj.workflow_job_template.execute_role
+
+    def filtered_queryset(self):
+        return self.model.objects.filter(
+            workflowjobtemplatenodes__workflow_job_template__in=WorkflowJobTemplate.accessible_pk_qs(
+                self.user, 'read_role'))
+
+
 for cls in BaseAccess.__subclasses__():
     access_registry[cls.model] = cls

awx/main/analytics/__init__.py

@@ -1 +1 @@
-from .core import register, gather, ship  # noqa
+from .core import register, gather, ship, table_version  # noqa

awx/main/analytics/collectors.py

@@ -12,14 +12,14 @@ from awx.main.utils import (get_awx_version, get_ansible_version,
                             get_custom_venv_choices, camelcase_to_underscore)
 from awx.main import models
 from django.contrib.sessions.models import Session
-from awx.main.analytics import register
+from awx.main.analytics import register, table_version
 
 '''
 This module is used to define metrics collected by awx.main.analytics.gather()
 Each function is decorated with a key name, and should return a data
 structure that can be serialized to JSON
 
-@register('something')
+@register('something', '1.0')
 def something(since):
     # the generated archive will contain a `something.json` w/ this JSON
     return {'some': 'json'}
@@ -31,7 +31,7 @@ data _since_ the last report date - i.e., new data in the last 24 hours)
 '''
 
 
-@register('config')
+@register('config', '1.0')
 def config(since):
     license_info = get_license(show_key=False)
     install_type = 'traditional'
@@ -62,7 +62,7 @@ def config(since):
     }
 
 
-@register('counts')
+@register('counts', '1.0')
 def counts(since):
     counts = {}
     for cls in (models.Organization, models.Team, models.User,
@@ -93,10 +93,11 @@ def counts(since):
     counts['active_user_sessions'] = active_user_sessions
     counts['active_anonymous_sessions'] = active_anonymous_sessions
     counts['running_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').filter(status__in=('running', 'waiting',)).count()
+    counts['pending_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').filter(status__in=('pending',)).count()
     return counts
 
     
-@register('org_counts')
+@register('org_counts', '1.0')
 def org_counts(since):
     counts = {}
     for org in models.Organization.objects.annotate(num_users=Count('member_role__members', distinct=True), 
@@ -108,7 +109,7 @@ def org_counts(since):
     return counts
     
     
-@register('cred_type_counts')
+@register('cred_type_counts', '1.0')
 def cred_type_counts(since):
     counts = {}
     for cred_type in models.CredentialType.objects.annotate(num_credentials=Count(
@@ -120,7 +121,7 @@ def cred_type_counts(since):
     return counts
     
     
-@register('inventory_counts')
+@register('inventory_counts', '1.0')
 def inventory_counts(since):
     counts = {}
     for inv in models.Inventory.objects.filter(kind='').annotate(num_sources=Count('inventory_sources', distinct=True), 
@@ -140,7 +141,7 @@ def inventory_counts(since):
     return counts
 
 
-@register('projects_by_scm_type')
+@register('projects_by_scm_type', '1.0')
 def projects_by_scm_type(since):
     counts = dict(
         (t[0] or 'manual', 0)
@@ -153,10 +154,16 @@ def projects_by_scm_type(since):
     return counts
 
 
-@register('instance_info')
-def instance_info(since):
+def _get_isolated_datetime(last_check):
+    if last_check:
+        return last_check.isoformat()
+    return last_check
+
+
+@register('instance_info', '1.0')
+def instance_info(since, include_hostnames=False):
     info = {}
-    instances = models.Instance.objects.values_list('hostname').annotate().values(
+    instances = models.Instance.objects.values_list('hostname').values(
         'uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'hostname', 'last_isolated_check', 'enabled')
     for instance in instances:
         instance_info = {
@@ -166,14 +173,16 @@ def instance_info(since):
             'cpu': instance['cpu'],
             'memory': instance['memory'],
             'managed_by_policy': instance['managed_by_policy'],
-            'last_isolated_check': instance['last_isolated_check'],
+            'last_isolated_check': _get_isolated_datetime(instance['last_isolated_check']),
             'enabled': instance['enabled']
         }
+        if include_hostnames is True:
+            instance_info['hostname'] = instance['hostname']
         info[instance['uuid']] = instance_info
     return info
 
 
-@register('job_counts')
+@register('job_counts', '1.0')
 def job_counts(since):
     counts = {}
     counts['total_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').count()
@@ -183,22 +192,34 @@ def job_counts(since):
     return counts
     
     
-@register('job_instance_counts')
+@register('job_instance_counts', '1.0')
 def job_instance_counts(since):
     counts = {}
     job_types = models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
-        'execution_node', 'launch_type').annotate(job_launch_type=Count('launch_type'))
+        'execution_node', 'launch_type').annotate(job_launch_type=Count('launch_type')).order_by()
     for job in job_types:
         counts.setdefault(job[0], {}).setdefault('launch_type', {})[job[1]] = job[2]
         
     job_statuses = models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
-        'execution_node', 'status').annotate(job_status=Count('status'))
+        'execution_node', 'status').annotate(job_status=Count('status')).order_by()
     for job in job_statuses:
         counts.setdefault(job[0], {}).setdefault('status', {})[job[1]] = job[2]
     return counts
 
 
+@register('query_info', '1.0')
+def query_info(since, collection_type):
+    query_info = {}
+    query_info['last_run'] = str(since)
+    query_info['current_time'] = str(now())
+    query_info['collection_type'] = collection_type
+    return query_info
+
+
 # Copies Job Events from db to a .csv to be shipped
+@table_version('events_table.csv', '1.0')
+@table_version('unified_jobs_table.csv', '1.0')
+@table_version('unified_job_template_table.csv', '1.0')
 def copy_tables(since, full_path):
     def _copy_table(table, query, path):
         file_path = os.path.join(path, table + '_table.csv')
@@ -227,10 +248,12 @@ def copy_tables(since, full_path):
                               WHERE main_jobevent.created > {}
                               ORDER BY main_jobevent.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'"))
     _copy_table(table='events', query=events_query, path=full_path)
-    
-    unified_job_query = '''COPY (SELECT main_unifiedjob.id, 
+
+    unified_job_query = '''COPY (SELECT main_unifiedjob.id,
                                  main_unifiedjob.polymorphic_ctype_id,
                                  django_content_type.model,
+                                 main_project.organization_id,
+                                 main_organization.name as organization_name,
                                  main_unifiedjob.created,  
                                  main_unifiedjob.name,  
                                  main_unifiedjob.unified_job_template_id, 
@@ -246,13 +269,16 @@ def copy_tables(since, full_path):
                                  main_unifiedjob.elapsed, 
                                  main_unifiedjob.job_explanation, 
                                  main_unifiedjob.instance_group_id
-                                 FROM main_unifiedjob, django_content_type
-                                 WHERE main_unifiedjob.created > {} AND 
-                                 main_unifiedjob.polymorphic_ctype_id = django_content_type.id AND 
-                                 main_unifiedjob.launch_type != 'sync'
+                                 FROM main_unifiedjob
+                                 JOIN main_job ON main_unifiedjob.id = main_job.unifiedjob_ptr_id
+                                 JOIN django_content_type ON main_unifiedjob.polymorphic_ctype_id = django_content_type.id
+                                 JOIN main_project ON main_project.unifiedjobtemplate_ptr_id = main_job.project_id
+                                 JOIN main_organization ON main_organization.id = main_project.organization_id
+                                 WHERE main_unifiedjob.created > {} 
+                                 AND main_unifiedjob.launch_type != 'sync'
                                  ORDER BY main_unifiedjob.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'"))    
     _copy_table(table='unified_jobs', query=unified_job_query, path=full_path)
-    
+
     unified_job_template_query = '''COPY (SELECT main_unifiedjobtemplate.id, 
                                  main_unifiedjobtemplate.polymorphic_ctype_id,
                                  django_content_type.model,
@@ -273,4 +299,3 @@ def copy_tables(since, full_path):
                                  ORDER BY main_unifiedjobtemplate.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'"))    
     _copy_table(table='unified_job_template', query=unified_job_template_query, path=full_path)
     return
-

awx/main/analytics/core.py

@@ -18,11 +18,13 @@ from awx.main.access import access_registry
 from awx.main.models.ha import TowerAnalyticsState
 
 
-__all__ = ['register', 'gather', 'ship']
+__all__ = ['register', 'gather', 'ship', 'table_version']
 
 
 logger = logging.getLogger('awx.main.analytics')
 
+manifest = dict()
+
 
 def _valid_license():
     try:
@@ -35,25 +37,37 @@ def _valid_license():
     return True
 
 
-def register(key):
+def register(key, version):
     """
     A decorator used to register a function as a metric collector.
 
     Decorated functions should return JSON-serializable objects.
 
-    @register('projects_by_scm_type')
+    @register('projects_by_scm_type', 1)
     def projects_by_scm_type():
         return {'git': 5, 'svn': 1, 'hg': 0}
     """
 
     def decorate(f):
         f.__awx_analytics_key__ = key
+        f.__awx_analytics_version__ = version
         return f
 
     return decorate
 
 
-def gather(dest=None, module=None):
+def table_version(file_name, version):
+
+    global manifest
+    manifest[file_name] = version
+
+    def decorate(f):
+        return f
+
+    return decorate
+
+
+def gather(dest=None, module=None, collection_type='scheduled'):
     """
     Gather all defined metrics and write them as JSON files in a .tgz
 
@@ -84,18 +98,33 @@ def gather(dest=None, module=None):
         from awx.main.analytics import collectors
         module = collectors
 
+
     dest = dest or tempfile.mkdtemp(prefix='awx_analytics')
     for name, func in inspect.getmembers(module):
         if inspect.isfunction(func) and hasattr(func, '__awx_analytics_key__'):
             key = func.__awx_analytics_key__
+            manifest['{}.json'.format(key)] = func.__awx_analytics_version__
             path = '{}.json'.format(os.path.join(dest, key))
             with open(path, 'w', encoding='utf-8') as f:
                 try:
-                    json.dump(func(last_run), f)
+                    if func.__name__ == 'query_info':
+                        json.dump(func(last_run, collection_type=collection_type), f)
+                    else:
+                        json.dump(func(last_run), f)
                 except Exception:
                     logger.exception("Could not generate metric {}.json".format(key))
                     f.close()
                     os.remove(f.name)
+    
+    path = os.path.join(dest, 'manifest.json')
+    with open(path, 'w', encoding='utf-8') as f:
+        try:
+            json.dump(manifest, f)
+        except Exception:
+            logger.exception("Could not generate manifest.json")
+            f.close()
+            os.remove(f.name)
+
     try:
         collectors.copy_tables(since=last_run, full_path=dest)
     except Exception:
@@ -119,24 +148,28 @@ def ship(path):
     """
     Ship gathered metrics via the Insights agent
     """
-    agent = 'insights-client'
-    if shutil.which(agent) is None:
-        logger.error('could not find {} on PATH'.format(agent))
-        return
-    logger.debug('shipping analytics file: {}'.format(path))
     try:
-        cmd = [
-            agent, '--payload', path, '--content-type', settings.INSIGHTS_AGENT_MIME
-        ]
-        output = smart_str(subprocess.check_output(cmd, timeout=60 * 5))
-        logger.debug(output)
-        # reset the `last_run` when data is shipped
-        run_now = now()
-        state = TowerAnalyticsState.get_solo()
-        state.last_run = run_now
-        state.save()
+        agent = 'insights-client'
+        if shutil.which(agent) is None:
+            logger.error('could not find {} on PATH'.format(agent))
+            return
+        logger.debug('shipping analytics file: {}'.format(path))
+        try:
+            cmd = [
+                agent, '--payload', path, '--content-type', settings.INSIGHTS_AGENT_MIME
+            ]
+            output = smart_str(subprocess.check_output(cmd, timeout=60 * 5))
+            logger.debug(output)
+            # reset the `last_run` when data is shipped
+            run_now = now()
+            state = TowerAnalyticsState.get_solo()
+            state.last_run = run_now
+            state.save()
 
-    except subprocess.CalledProcessError:
-        logger.exception('{} failure:'.format(cmd))
-    except subprocess.TimeoutExpired:
-        logger.exception('{} timeout:'.format(cmd))
+        except subprocess.CalledProcessError:
+            logger.exception('{} failure:'.format(cmd))
+        except subprocess.TimeoutExpired:
+            logger.exception('{} timeout:'.format(cmd))
+    finally:
+        # cleanup tar.gz
+        os.remove(path)

awx/main/analytics/metrics.py

@@ -15,6 +15,7 @@ from awx.main.analytics.collectors import (
     counts, 
     instance_info,
     job_instance_counts,
+    job_counts,
 )
 
 
@@ -36,11 +37,13 @@ INV_SCRIPT_COUNT = Gauge('awx_inventory_scripts_total', 'Number of invetory scri
 USER_SESSIONS = Gauge('awx_sessions_total', 'Number of sessions', ['type',])
 CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs')
 RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the Tower system')
+PENDING_JOBS = Gauge('awx_pending_jobs_total', 'Number of pending jobs on the Tower system')
+STATUS = Gauge('awx_status_total', 'Status of Job launched', ['status',])
 
-INSTANCE_CAPACITY = Gauge('awx_instance_capacity', 'Capacity of each node in a Tower system', ['instance_uuid',])
-INSTANCE_CPU = Gauge('awx_instance_cpu', 'CPU cores on each node in a Tower system', ['instance_uuid',])
-INSTANCE_MEMORY = Gauge('awx_instance_memory', 'RAM (Kb) on each node in a Tower system', ['instance_uuid',])
-INSTANCE_INFO = Info('awx_instance', 'Info about each node in a Tower system', ['instance_uuid',])
+INSTANCE_CAPACITY = Gauge('awx_instance_capacity', 'Capacity of each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_CPU = Gauge('awx_instance_cpu', 'CPU cores on each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_MEMORY = Gauge('awx_instance_memory', 'RAM (Kb) on each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_INFO = Info('awx_instance', 'Info about each node in a Tower system', ['hostname', 'instance_uuid',])
 INSTANCE_LAUNCH_TYPE = Gauge('awx_instance_launch_type_total', 'Type of Job launched', ['node', 'launch_type',])
 INSTANCE_STATUS = Gauge('awx_instance_status_total', 'Status of Job launched', ['node', 'status',])
 
@@ -87,15 +90,21 @@ def metrics():
     USER_SESSIONS.labels(type='user').set(current_counts['active_user_sessions'])
     USER_SESSIONS.labels(type='anonymous').set(current_counts['active_anonymous_sessions'])
 
+    all_job_data = job_counts(None)
+    statuses = all_job_data.get('status', {})
+    for status, value in statuses.items():
+        STATUS.labels(status=status).set(value)
+
     RUNNING_JOBS.set(current_counts['running_jobs'])
+    PENDING_JOBS.set(current_counts['pending_jobs'])
 
-
-    instance_data = instance_info(None)
-    for uuid in instance_data:
-        INSTANCE_CAPACITY.labels(instance_uuid=uuid).set(instance_data[uuid]['capacity'])
-        INSTANCE_CPU.labels(instance_uuid=uuid).set(instance_data[uuid]['cpu'])
-        INSTANCE_MEMORY.labels(instance_uuid=uuid).set(instance_data[uuid]['memory'])
-        INSTANCE_INFO.labels(instance_uuid=uuid).info({
+    instance_data = instance_info(None, include_hostnames=True)
+    for uuid, info in instance_data.items():
+        hostname = info['hostname']
+        INSTANCE_CAPACITY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['capacity'])
+        INSTANCE_CPU.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['cpu'])
+        INSTANCE_MEMORY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['memory'])
+        INSTANCE_INFO.labels(hostname=hostname, instance_uuid=uuid).info({
             'enabled': str(instance_data[uuid]['enabled']),
             'last_isolated_check': getattr(instance_data[uuid], 'last_isolated_check', 'None'),
             'managed_by_policy': str(instance_data[uuid]['managed_by_policy']),

awx/main/conf.py

@@ -124,6 +124,44 @@ register(
     category_slug='system',
 )
 
+register(
+    'REDHAT_USERNAME',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    encrypted=False,
+    read_only=False,
+    label=_('Red Hat customer username'),
+    help_text=_('This username is used to retrieve license information and to send Automation Analytics'),  # noqa
+    category=_('System'),
+    category_slug='system',
+)
+
+register(
+    'REDHAT_PASSWORD',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    encrypted=True,
+    read_only=False,
+    label=_('Red Hat customer password'),
+    help_text=_('This password is used to retrieve license information and to send Automation Analytics'),  # noqa
+    category=_('System'),
+    category_slug='system',
+)
+
+register(
+    'AUTOMATION_ANALYTICS_URL',
+    field_class=fields.URLField,
+    default='https://cloud.redhat.com',
+    schemes=('http', 'https'),
+    allow_plain_hostname=True,  # Allow hostname only without TLD.
+    label=_('Automation Analytics upload URL.'),
+    help_text=_('This setting is used to to configure data collection for the Automation Analytics dashboard'),
+    category=_('System'),
+    category_slug='system',
+)
+
 register(
     'INSTALL_UUID',
     field_class=fields.CharField,
@@ -328,6 +366,16 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'AWX_COLLECTIONS_ENABLED',
+    field_class=fields.BooleanField,
+    default=True,
+    label=_('Enable Collection(s) Download'),
+    help_text=_('Allows collections to be dynamically downloaded from a requirements.yml file for SCM projects.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
 register(
     'STDOUT_MAX_BYTES_DISPLAY',
     field_class=fields.IntegerField,

awx/main/consumers.py

@@ -24,7 +24,7 @@ def ws_connect(message):
     headers = dict(message.content.get('headers', ''))
     message.reply_channel.send({"accept": True})
     message.content['method'] = 'FAKE'
-    if message.user.is_authenticated():
+    if message.user.is_authenticated:
         message.reply_channel.send(
             {"text": json.dumps({"accept": True, "user": message.user.id})}
         )

awx/main/credential_plugins/aim.py

@@ -1,14 +1,14 @@
 from .plugin import CredentialPlugin
 
-import os
-import stat
-import tempfile
-import threading
 from urllib.parse import quote, urlencode, urljoin
 
 from django.utils.translation import ugettext_lazy as _
 import requests
 
+# AWX
+from awx.main.utils import (
+    create_temporary_fifo,
+)
 
 aim_inputs = {
     'fields': [{
@@ -60,24 +60,6 @@ aim_inputs = {
 }
 
 
-def create_temporary_fifo(data):
-    """Open fifo named pipe in a new thread using a temporary file path. The
-    thread blocks until data is read from the pipe.
-
-    Returns the path to the fifo.
-
-    :param data(bytes): Data to write to the pipe.
-    """
-    path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names()))
-    os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR)
-
-    threading.Thread(
-        target=lambda p, d: open(p, 'wb').write(d),
-        args=(path, data)
-    ).start()
-    return path
-
-
 def aim_backend(**kwargs):
     url = kwargs['url']
     client_cert = kwargs.get('client_cert', None)

awx/main/credential_plugins/conjur.py

@@ -1,15 +1,16 @@
 from .plugin import CredentialPlugin
 
 import base64
-import os
-import stat
-import tempfile
-import threading
 from urllib.parse import urljoin, quote_plus
 
 from django.utils.translation import ugettext_lazy as _
 import requests
 
+# AWX
+from awx.main.utils import (
+    create_temporary_fifo,
+)
+
 
 conjur_inputs = {
     'fields': [{
@@ -51,24 +52,6 @@ conjur_inputs = {
 }
 
 
-def create_temporary_fifo(data):
-    """Open fifo named pipe in a new thread using a temporary file path. The
-    thread blocks until data is read from the pipe.
-
-    Returns the path to the fifo.
-
-    :param data(bytes): Data to write to the pipe.
-    """
-    path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names()))
-    os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR)
-
-    threading.Thread(
-        target=lambda p, d: open(p, 'wb').write(d),
-        args=(path, data)
-    ).start()
-    return path
-
-
 def conjur_backend(**kwargs):
     url = kwargs['url']
     api_key = kwargs['api_key']

awx/main/credential_plugins/hashivault.py

@@ -8,6 +8,10 @@ from .plugin import CredentialPlugin
 import requests
 from django.utils.translation import ugettext_lazy as _
 
+# AWX
+from awx.main.utils import (
+    create_temporary_fifo,
+)
 
 base_inputs = {
     'fields': [{
@@ -22,12 +26,18 @@ base_inputs = {
         'type': 'string',
         'secret': True,
         'help_text': _('The access token used to authenticate to the Vault server'),
+    }, {
+        'id': 'cacert',
+        'label': _('CA Certificate'),
+        'type': 'string',
+        'multiline': True,
+        'help_text': _('The CA certificate used to verify the SSL certificate of the Vault server')
     }],
     'metadata': [{
         'id': 'secret_path',
         'label': _('Path to Secret'),
         'type': 'string',
-        'help_text': _('The path to the secret e.g., /some-engine/some-secret/'),
+        'help_text': _('The path to the secret stored in the secret backend e.g, /some/secret/')
     }],
     'required': ['url', 'token', 'secret_path'],
 }
@@ -40,7 +50,12 @@ hashi_kv_inputs['fields'].append({
     'help_text': _('API v1 is for static key/value lookups.  API v2 is for versioned key/value lookups.'),
     'default': 'v1',
 })
-hashi_kv_inputs['metadata'].extend([{
+hashi_kv_inputs['metadata'] = [{
+    'id': 'secret_backend',
+    'label': _('Name of Secret Backend'),
+    'type': 'string',
+    'help_text': _('The name of the kv secret backend (if left empty, the first segment of the secret path will be used).')
+}] + hashi_kv_inputs['metadata'] + [{
     'id': 'secret_key',
     'label': _('Key Name'),
     'type': 'string',
@@ -50,7 +65,7 @@ hashi_kv_inputs['metadata'].extend([{
     'label': _('Secret Version (v2 only)'),
     'type': 'string',
     'help_text': _('Used to specify a specific secret version (if left empty, the latest version will be used).'),
-}])
+}]
 hashi_kv_inputs['required'].extend(['api_version', 'secret_key'])
 
 hashi_ssh_inputs = copy.deepcopy(base_inputs)
@@ -75,36 +90,46 @@ hashi_ssh_inputs['required'].extend(['public_key', 'role'])
 
 def kv_backend(**kwargs):
     token = kwargs['token']
-    url = urljoin(kwargs['url'], 'v1')
+    url = kwargs['url']
     secret_path = kwargs['secret_path']
+    secret_backend = kwargs.get('secret_backend', None)
     secret_key = kwargs.get('secret_key', None)
-
+    cacert = kwargs.get('cacert', None)
     api_version = kwargs['api_version']
 
+    request_kwargs = {'timeout': 30}
+    if cacert:
+        request_kwargs['verify'] = create_temporary_fifo(cacert.encode())
+
     sess = requests.Session()
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
+
     if api_version == 'v2':
-        params = {}
         if kwargs.get('secret_version'):
-            params['version'] = kwargs['secret_version']
-        try:
-            mount_point, *path = pathlib.Path(secret_path.lstrip(os.sep)).parts
-            '/'.join(*path)
-        except Exception:
-            mount_point, path = secret_path, []
-        # https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version
-        response = sess.get(
-            '/'.join([url, mount_point, 'data'] + path).rstrip('/'),
-            params=params,
-            timeout=30
-        )
-        response.raise_for_status()
-        json = response.json()['data']
+            request_kwargs['params'] = {'version': kwargs['secret_version']}
+        if secret_backend:
+            path_segments = [secret_backend, 'data', secret_path]
+        else:
+            try:
+                mount_point, *path = pathlib.Path(secret_path.lstrip(os.sep)).parts
+                '/'.join(path)
+            except Exception:
+                mount_point, path = secret_path, []
+            # https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version
+            path_segments = [mount_point, 'data'] + path
     else:
-        # https://www.vaultproject.io/api/secret/kv/kv-v1.html#read-secret
-        response = sess.get('/'.join([url, secret_path]).rstrip('/'), timeout=30)
-        response.raise_for_status()
-        json = response.json()
+        if secret_backend:
+            path_segments = [secret_backend, secret_path]
+        else:
+            path_segments = [secret_path]
+
+    request_url = urljoin(url, '/'.join(['v1'] + path_segments)).rstrip('/')
+    response = sess.get(request_url, **request_kwargs)
+    response.raise_for_status()
+
+    json = response.json()
+    if api_version == 'v2':
+        json = json['data']
 
     if secret_key:
         try:
@@ -121,20 +146,22 @@ def ssh_backend(**kwargs):
     url = urljoin(kwargs['url'], 'v1')
     secret_path = kwargs['secret_path']
     role = kwargs['role']
+    cacert = kwargs.get('cacert', None)
+
+    request_kwargs = {'timeout': 30}
+    if cacert:
+        request_kwargs['verify'] = create_temporary_fifo(cacert.encode())
+
+    request_kwargs['json'] = {'public_key': kwargs['public_key']}
+    if kwargs.get('valid_principals'):
+        request_kwargs['json']['valid_principals'] = kwargs['valid_principals']
 
     sess = requests.Session()
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
-    json = {
-        'public_key': kwargs['public_key']
-    }
-    if kwargs.get('valid_principals'):
-        json['valid_principals'] = kwargs['valid_principals']
     # https://www.vaultproject.io/api/secret/ssh/index.html#sign-ssh-key
-    resp = sess.post(
-        '/'.join([url, secret_path, 'sign', role]).rstrip('/'),
-        json=json,
-        timeout=30
-    )
+    request_url = '/'.join([url, secret_path, 'sign', role]).rstrip('/')
+    resp = sess.post(request_url, **request_kwargs)
+
     resp.raise_for_status()
     return resp.json()['data']['signed_key']
 

awx/main/fields.py

@@ -11,8 +11,9 @@ from jinja2 import Environment, StrictUndefined
 from jinja2.exceptions import UndefinedError, TemplateSyntaxError
 
 # Django
-import django
+from django.contrib.postgres.fields import JSONField as upstream_JSONBField
 from django.core import exceptions as django_exceptions
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db.models.signals import (
     post_save,
     post_delete,
@@ -37,7 +38,6 @@ import jsonschema.exceptions
 
 # Django-JSONField
 from jsonfield import JSONField as upstream_JSONField
-from jsonbfield.fields import JSONField as upstream_JSONBField
 
 # DRF
 from rest_framework import serializers
@@ -76,10 +76,10 @@ class JSONField(upstream_JSONField):
     def db_type(self, connection):
         return 'text'
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         if value in {'', None} and not self.null:
             return {}
-        return super(JSONField, self).from_db_value(value, expression, connection, context)
+        return super(JSONField, self).from_db_value(value, expression, connection)
 
 
 class JSONBField(upstream_JSONBField):
@@ -91,12 +91,12 @@ class JSONBField(upstream_JSONBField):
     def get_db_prep_value(self, value, connection, prepared=False):
         if connection.vendor == 'sqlite':
             # sqlite (which we use for tests) does not support jsonb;
-            return json.dumps(value)
+            return json.dumps(value, cls=DjangoJSONEncoder)
         return super(JSONBField, self).get_db_prep_value(
             value, connection, prepared
         )
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         # Work around a bug in django-jsonfield
         # https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos
         if isinstance(value, str):
@@ -112,14 +112,9 @@ class AutoSingleRelatedObjectDescriptor(ReverseOneToOneDescriptor):
 
     def __get__(self, instance, instance_type=None):
         try:
-            return super(AutoSingleRelatedObjectDescriptor,
-                         self).__get__(instance, instance_type)
+            return super(AutoSingleRelatedObjectDescriptor, self).__get__(instance, instance_type)
         except self.related.related_model.DoesNotExist:
             obj = self.related.related_model(**{self.related.field.name: instance})
-            if self.related.field.rel.parent_link:
-                raise NotImplementedError('not supported with polymorphic!')
-                for f in instance._meta.local_fields:
-                    setattr(obj, f.name, getattr(instance, f.name))
             obj.save()
             return obj
 
@@ -169,7 +164,7 @@ def is_implicit_parent(parent_role, child_role):
         # The only singleton implicit parent is the system admin being
         # a parent of the system auditor role
         return bool(
-            child_role.singleton_name == ROLE_SINGLETON_SYSTEM_AUDITOR and 
+            child_role.singleton_name == ROLE_SINGLETON_SYSTEM_AUDITOR and
             parent_role.singleton_name == ROLE_SINGLETON_SYSTEM_ADMINISTRATOR
         )
     # Get the list of implicit parents that were defined at the class level.
@@ -453,21 +448,6 @@ class JSONSchemaField(JSONBField):
                 params={'value': value},
             )
 
-    def get_db_prep_value(self, value, connection, prepared=False):
-        if connection.vendor == 'sqlite':
-            # sqlite (which we use for tests) does not support jsonb;
-            return json.dumps(value)
-        return super(JSONSchemaField, self).get_db_prep_value(
-            value, connection, prepared
-        )
-
-    def from_db_value(self, value, expression, connection, context):
-        # Work around a bug in django-jsonfield
-        # https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos
-        if isinstance(value, str):
-            return json.loads(value)
-        return value
-
 
 @JSONSchemaField.format_checker.checks('vault_id')
 def format_vault_id(value):
@@ -711,10 +691,12 @@ class CredentialInputField(JSONSchemaField):
 
             if model_instance.has_encrypted_ssh_key_data and not value.get('ssh_key_unlock'):
                 errors['ssh_key_unlock'] = [_('must be set when SSH key is encrypted.')]
+            
             if all([
                 model_instance.inputs.get('ssh_key_data'),
                 value.get('ssh_key_unlock'),
-                not model_instance.has_encrypted_ssh_key_data
+                not model_instance.has_encrypted_ssh_key_data,
+                'ssh_key_data' not in errors
             ]):
                 errors['ssh_key_unlock'] = [_('should not be set when SSH key is not encrypted.')]
 
@@ -986,7 +968,7 @@ class OAuth2ClientSecretField(models.CharField):
             encrypt_value(value), connection, prepared
         )
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         if value and value.startswith('$encrypted$'):
             return decrypt_value(get_encryption_key('value', pk=None), value)
         return value
@@ -1022,38 +1004,6 @@ class OrderedManyToManyDescriptor(ManyToManyDescriptor):
                         '%s__position' % self.through._meta.model_name
                     )
 
-                def add(self, *objs):
-                    # Django < 2 doesn't support this method on
-                    # ManyToManyFields w/ an intermediary model
-                    # We should be able to remove this code snippet when we
-                    # upgrade Django.
-                    # see: https://github.com/django/django/blob/stable/1.11.x/django/db/models/fields/related_descriptors.py#L926
-                    if not django.__version__.startswith('1.'):
-                        raise RuntimeError(
-                            'This method is no longer necessary in Django>=2'
-                        )
-                    try:
-                        self.through._meta.auto_created = True
-                        super(OrderedManyRelatedManager, self).add(*objs)
-                    finally:
-                        self.through._meta.auto_created = False
-
-                def remove(self, *objs):
-                    # Django < 2 doesn't support this method on
-                    # ManyToManyFields w/ an intermediary model
-                    # We should be able to remove this code snippet when we
-                    # upgrade Django.
-                    # see: https://github.com/django/django/blob/stable/1.11.x/django/db/models/fields/related_descriptors.py#L944
-                    if not django.__version__.startswith('1.'):
-                        raise RuntimeError(
-                            'This method is no longer necessary in Django>=2'
-                        )
-                    try:
-                        self.through._meta.auto_created = True
-                        super(OrderedManyRelatedManager, self).remove(*objs)
-                    finally:
-                        self.through._meta.auto_created = False
-
             return OrderedManyRelatedManager
 
         return add_custom_queryset_to_many_related_manager(

awx/main/management/commands/check_license.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved
 
+import json
+
 from awx.main.utils import get_licenser
 from django.core.management.base import BaseCommand
 
@@ -8,6 +10,15 @@ from django.core.management.base import BaseCommand
 class Command(BaseCommand):
     """Returns license type, e.g., 'enterprise', 'open', 'none'"""
 
+    def add_arguments(self, parser):
+        parser.add_argument('--data', dest='data', action='store_true',
+                            help='verbose, prints the actual (sanitized) license')
+
     def handle(self, *args, **options):
         super(Command, self).__init__()
-        return get_licenser().validate().get('license_type', 'none')
+        license = get_licenser().validate()
+        if options.get('data'):
+            if license.get('license_key', '') != 'UNLICENSED':
+                license['license_key'] = '********'
+            return json.dumps(license)
+        return license.get('license_type', 'none')

awx/main/management/commands/gather_analytics.py

@@ -23,7 +23,7 @@ class Command(BaseCommand):
         self.logger.propagate = False
 
     def handle(self, *args, **options):
-        tgz = gather()
+        tgz = gather(collection_type='manual')
         self.init_logging()
         if tgz:
             self.logger.debug(tgz)

awx/main/management/commands/run_dispatcher.py

@@ -10,6 +10,7 @@ from django.core.management.base import BaseCommand
 from django.db import connection as django_connection, connections
 from kombu import Exchange, Queue
 
+from awx.main.utils.handlers import AWXProxyHandler
 from awx.main.dispatch import get_local_queuename, reaper
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.kombu import Connection
@@ -121,6 +122,12 @@ class Command(BaseCommand):
 
         reaper.reap()
         consumer = None
+
+        # don't ship external logs inside the dispatcher's parent process
+        # this exists to work around a race condition + deadlock bug on fork
+        # in cpython itself:
+        # https://bugs.python.org/issue37429
+        AWXProxyHandler.disable()
         with Connection(settings.BROKER_URL) as conn:
             try:
                 bcast = 'tower_broadcast_all'

awx/main/middleware.py

@@ -73,7 +73,7 @@ class ActivityStreamMiddleware(threading.local, MiddlewareMixin):
         super().__init__(get_response)
 
     def process_request(self, request):
-        if hasattr(request, 'user') and hasattr(request.user, 'is_authenticated') and request.user.is_authenticated():
+        if hasattr(request, 'user') and request.user.is_authenticated:
             user = request.user
         else:
             user = None

awx/main/migrations/0001_initial.py

@@ -44,7 +44,7 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('host_name', models.CharField(default='', max_length=1024, editable=False)),
                 ('event', models.CharField(max_length=100, choices=[('runner_on_failed', 'Host Failed'), ('runner_on_ok', 'Host OK'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_skipped', 'Host Skipped')])),
-                ('event_data', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('counter', models.PositiveIntegerField(default=0)),
@@ -62,7 +62,7 @@ class Migration(migrations.Migration):
                 ('expires', models.DateTimeField(default=django.utils.timezone.now)),
                 ('request_hash', models.CharField(default='', max_length=40, blank=True)),
                 ('reason', models.CharField(default='', help_text='Reason the auth token was invalidated.', max_length=1024, blank=True)),
-                ('user', models.ForeignKey(related_name='auth_tokens', to=settings.AUTH_USER_MODEL)),
+                ('user', models.ForeignKey(related_name='auth_tokens', on_delete=models.CASCADE, to=settings.AUTH_USER_MODEL)),
             ],
         ),
         migrations.CreateModel(
@@ -198,7 +198,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('event', models.CharField(max_length=100, choices=[('runner_on_failed', 'Host Failed'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete')])),
-                ('event_data', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('host_name', models.CharField(default='', max_length=1024, editable=False)),
@@ -241,7 +241,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('created', models.DateTimeField(auto_now_add=True)),
                 ('modified', models.DateTimeField(auto_now=True)),
-                ('instance', models.ForeignKey(to='main.Instance')),
+                ('instance', models.ForeignKey(on_delete=models.CASCADE, to='main.Instance')),
             ],
         ),
         migrations.CreateModel(
@@ -287,7 +287,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('ldap_dn', models.CharField(default='', max_length=1024)),
-                ('user', awx.main.fields.AutoOneToOneField(related_name='profile', editable=False, to=settings.AUTH_USER_MODEL)),
+                ('user', awx.main.fields.AutoOneToOneField(related_name='profile', editable=False, on_delete=models.CASCADE, to=settings.AUTH_USER_MODEL)),
             ],
         ),
         migrations.CreateModel(
@@ -304,7 +304,7 @@ class Migration(migrations.Migration):
                 ('dtend', models.DateTimeField(default=None, null=True, editable=False)),
                 ('rrule', models.CharField(max_length=255)),
                 ('next_run', models.DateTimeField(default=None, null=True, editable=False)),
-                ('extra_data', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('extra_data', jsonfield.fields.JSONField(default=dict, blank=True)),
                 ('created_by', models.ForeignKey(related_name="{u'class': 'schedule', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('modified_by', models.ForeignKey(related_name="{u'class': 'schedule', u'app_label': 'main'}(class)s_modified+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')),
@@ -343,7 +343,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=512)),
                 ('old_pk', models.PositiveIntegerField(default=None, null=True, editable=False)),
                 ('launch_type', models.CharField(default='manual', max_length=20, editable=False, choices=[('manual', 'Manual'), ('relaunch', 'Relaunch'), ('callback', 'Callback'), ('scheduled', 'Scheduled'), ('dependency', 'Dependency')])),
-                ('cancel_flag', models.BooleanField(default=False, editable=False)),
+                ('cancel_flag', models.BooleanField(blank=True, default=False, editable=False)),
                 ('status', models.CharField(default='new', max_length=20, editable=False, choices=[('new', 'New'), ('pending', 'Pending'), ('waiting', 'Waiting'), ('running', 'Running'), ('successful', 'Successful'), ('failed', 'Failed'), ('error', 'Error'), ('canceled', 'Canceled')])),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('started', models.DateTimeField(default=None, null=True, editable=False)),
@@ -351,7 +351,7 @@ class Migration(migrations.Migration):
                 ('elapsed', models.DecimalField(editable=False, max_digits=12, decimal_places=3)),
                 ('job_args', models.TextField(default='', editable=False, blank=True)),
                 ('job_cwd', models.CharField(default='', max_length=1024, editable=False, blank=True)),
-                ('job_env', jsonfield.fields.JSONField(default={}, editable=False, blank=True)),
+                ('job_env', jsonfield.fields.JSONField(default=dict, editable=False, blank=True)),
                 ('job_explanation', models.TextField(default='', editable=False, blank=True)),
                 ('start_args', models.TextField(default='', editable=False, blank=True)),
                 ('result_stdout_text', models.TextField(default='', editable=False, blank=True)),
@@ -380,7 +380,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='AdHocCommand',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('job_type', models.CharField(default='run', max_length=64, choices=[('run', 'Run'), ('check', 'Check')])),
                 ('limit', models.CharField(default='', max_length=1024, blank=True)),
                 ('module_name', models.CharField(default='', max_length=1024, blank=True)),
@@ -394,7 +394,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='InventorySource',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('source', models.CharField(default='', max_length=32, blank=True, choices=[('', 'Manual'), ('file', 'Local File, Directory or Script'), ('rax', 'Rackspace Cloud Servers'), ('ec2', 'Amazon EC2'), ('gce', 'Google Compute Engine'), ('azure', 'Microsoft Azure'), ('vmware', 'VMware vCenter'), ('openstack', 'OpenStack'), ('custom', 'Custom Script')])),
                 ('source_path', models.CharField(default='', max_length=1024, editable=False, blank=True)),
                 ('source_vars', models.TextField(default='', help_text='Inventory source variables in YAML or JSON format.', blank=True)),
@@ -411,7 +411,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='InventoryUpdate',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('source', models.CharField(default='', max_length=32, blank=True, choices=[('', 'Manual'), ('file', 'Local File, Directory or Script'), ('rax', 'Rackspace Cloud Servers'), ('ec2', 'Amazon EC2'), ('gce', 'Google Compute Engine'), ('azure', 'Microsoft Azure'), ('vmware', 'VMware vCenter'), ('openstack', 'OpenStack'), ('custom', 'Custom Script')])),
                 ('source_path', models.CharField(default='', max_length=1024, editable=False, blank=True)),
                 ('source_vars', models.TextField(default='', help_text='Inventory source variables in YAML or JSON format.', blank=True)),
@@ -427,7 +427,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Job',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('job_type', models.CharField(default='run', max_length=64, choices=[('run', 'Run'), ('check', 'Check'), ('scan', 'Scan')])),
                 ('playbook', models.CharField(default='', max_length=1024, blank=True)),
                 ('forks', models.PositiveIntegerField(default=0, blank=True)),
@@ -435,7 +435,7 @@ class Migration(migrations.Migration):
                 ('verbosity', models.PositiveIntegerField(default=0, blank=True, choices=[(0, '0 (Normal)'), (1, '1 (Verbose)'), (2, '2 (More Verbose)'), (3, '3 (Debug)'), (4, '4 (Connection Debug)'), (5, '5 (WinRM Debug)')])),
                 ('extra_vars', models.TextField(default='', blank=True)),
                 ('job_tags', models.CharField(default='', max_length=1024, blank=True)),
-                ('force_handlers', models.BooleanField(default=False)),
+                ('force_handlers', models.BooleanField(blank=True, default=False)),
                 ('skip_tags', models.CharField(default='', max_length=1024, blank=True)),
                 ('start_at_task', models.CharField(default='', max_length=1024, blank=True)),
                 ('become_enabled', models.BooleanField(default=False)),
@@ -448,7 +448,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='JobTemplate',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('job_type', models.CharField(default='run', max_length=64, choices=[('run', 'Run'), ('check', 'Check'), ('scan', 'Scan')])),
                 ('playbook', models.CharField(default='', max_length=1024, blank=True)),
                 ('forks', models.PositiveIntegerField(default=0, blank=True)),
@@ -456,14 +456,14 @@ class Migration(migrations.Migration):
                 ('verbosity', models.PositiveIntegerField(default=0, blank=True, choices=[(0, '0 (Normal)'), (1, '1 (Verbose)'), (2, '2 (More Verbose)'), (3, '3 (Debug)'), (4, '4 (Connection Debug)'), (5, '5 (WinRM Debug)')])),
                 ('extra_vars', models.TextField(default='', blank=True)),
                 ('job_tags', models.CharField(default='', max_length=1024, blank=True)),
-                ('force_handlers', models.BooleanField(default=False)),
+                ('force_handlers', models.BooleanField(blank=True, default=False)),
                 ('skip_tags', models.CharField(default='', max_length=1024, blank=True)),
                 ('start_at_task', models.CharField(default='', max_length=1024, blank=True)),
                 ('become_enabled', models.BooleanField(default=False)),
                 ('host_config_key', models.CharField(default='', max_length=1024, blank=True)),
                 ('ask_variables_on_launch', models.BooleanField(default=False)),
                 ('survey_enabled', models.BooleanField(default=False)),
-                ('survey_spec', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('survey_spec', jsonfield.fields.JSONField(default=dict, blank=True)),
             ],
             options={
                 'ordering': ('name',),
@@ -473,7 +473,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Project',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('local_path', models.CharField(help_text='Local path (relative to PROJECTS_ROOT) containing playbooks and related files for this project.', max_length=1024, blank=True)),
                 ('scm_type', models.CharField(default='', max_length=8, verbose_name='SCM Type', blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion')])),
                 ('scm_url', models.CharField(default='', max_length=1024, verbose_name='SCM URL', blank=True)),
@@ -492,7 +492,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='ProjectUpdate',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('local_path', models.CharField(help_text='Local path (relative to PROJECTS_ROOT) containing playbooks and related files for this project.', max_length=1024, blank=True)),
                 ('scm_type', models.CharField(default='', max_length=8, verbose_name='SCM Type', blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion')])),
                 ('scm_url', models.CharField(default='', max_length=1024, verbose_name='SCM URL', blank=True)),
@@ -505,7 +505,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='SystemJob',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('job_type', models.CharField(default='', max_length=32, blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('cleanup_deleted', 'Purge previously deleted items from the database'), ('cleanup_facts', 'Purge and/or reduce the granularity of system tracking data')])),
                 ('extra_vars', models.TextField(default='', blank=True)),
             ],
@@ -517,7 +517,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='SystemJobTemplate',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('job_type', models.CharField(default='', max_length=32, blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('cleanup_deleted', 'Purge previously deleted items from the database'), ('cleanup_facts', 'Purge and/or reduce the granularity of system tracking data')])),
             ],
             bases=('main.unifiedjobtemplate', models.Model),
@@ -550,7 +550,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjobtemplate',
             name='polymorphic_ctype',
-            field=models.ForeignKey(related_name='polymorphic_main.unifiedjobtemplate_set+', editable=False, to='contenttypes.ContentType', null=True),
+            field=models.ForeignKey(related_name='polymorphic_main.unifiedjobtemplate_set+', editable=False, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', null=True),
         ),
         migrations.AddField(
             model_name='unifiedjobtemplate',
@@ -575,7 +575,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjob',
             name='polymorphic_ctype',
-            field=models.ForeignKey(related_name='polymorphic_main.unifiedjob_set+', editable=False, to='contenttypes.ContentType', null=True),
+            field=models.ForeignKey(related_name='polymorphic_main.unifiedjob_set+', editable=False, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', null=True),
         ),
         migrations.AddField(
             model_name='unifiedjob',
@@ -595,7 +595,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='unified_job_template',
-            field=models.ForeignKey(related_name='schedules', to='main.UnifiedJobTemplate'),
+            field=models.ForeignKey(related_name='schedules', on_delete=django.db.models.deletion.CASCADE, to='main.UnifiedJobTemplate'),
         ),
         migrations.AddField(
             model_name='permission',
@@ -610,12 +610,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='joborigin',
             name='unified_job',
-            field=models.OneToOneField(related_name='job_origin', to='main.UnifiedJob'),
+            field=models.OneToOneField(related_name='job_origin', on_delete=django.db.models.deletion.CASCADE, to='main.UnifiedJob'),
         ),
         migrations.AddField(
             model_name='inventory',
             name='organization',
-            field=models.ForeignKey(related_name='inventories', to='main.Organization', help_text='Organization containing this inventory.'),
+            field=models.ForeignKey(related_name='inventories', on_delete=django.db.models.deletion.CASCADE, to='main.Organization', help_text='Organization containing this inventory.'),
         ),
         migrations.AddField(
             model_name='inventory',
@@ -625,7 +625,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='host',
             name='inventory',
-            field=models.ForeignKey(related_name='hosts', to='main.Inventory'),
+            field=models.ForeignKey(related_name='hosts', on_delete=django.db.models.deletion.CASCADE, to='main.Inventory'),
         ),
         migrations.AddField(
             model_name='host',
@@ -650,7 +650,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='group',
             name='inventory',
-            field=models.ForeignKey(related_name='groups', to='main.Inventory'),
+            field=models.ForeignKey(related_name='groups', on_delete=django.db.models.deletion.CASCADE, to='main.Inventory'),
         ),
         migrations.AddField(
             model_name='group',
@@ -680,12 +680,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='credential',
             name='team',
-            field=models.ForeignKey(related_name='credentials', default=None, blank=True, to='main.Team', null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.Team', null=True),
         ),
         migrations.AddField(
             model_name='credential',
             name='user',
-            field=models.ForeignKey(related_name='credentials', default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
         ),
         migrations.AddField(
             model_name='adhoccommandevent',
@@ -774,7 +774,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='projectupdate',
             name='project',
-            field=models.ForeignKey(related_name='project_updates', editable=False, to='main.Project'),
+            field=models.ForeignKey(related_name='project_updates', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.Project'),
         ),
         migrations.AddField(
             model_name='project',
@@ -814,12 +814,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='jobhostsummary',
             name='job',
-            field=models.ForeignKey(related_name='job_host_summaries', editable=False, to='main.Job'),
+            field=models.ForeignKey(related_name='job_host_summaries', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.Job'),
         ),
         migrations.AddField(
             model_name='jobevent',
             name='job',
-            field=models.ForeignKey(related_name='job_events', editable=False, to='main.Job'),
+            field=models.ForeignKey(related_name='job_events', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.Job'),
         ),
         migrations.AddField(
             model_name='job',
@@ -859,7 +859,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventoryupdate',
             name='inventory_source',
-            field=models.ForeignKey(related_name='inventory_updates', editable=False, to='main.InventorySource'),
+            field=models.ForeignKey(related_name='inventory_updates', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.InventorySource'),
         ),
         migrations.AddField(
             model_name='inventoryupdate',
@@ -874,12 +874,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventorysource',
             name='group',
-            field=awx.main.fields.AutoOneToOneField(related_name='inventory_source', null=True, default=None, editable=False, to='main.Group'),
+            field=awx.main.fields.AutoOneToOneField(related_name='inventory_source', on_delete=django.db.models.deletion.SET_NULL, null=True, default=None, editable=False, to='main.Group'),
         ),
         migrations.AddField(
             model_name='inventorysource',
             name='inventory',
-            field=models.ForeignKey(related_name='inventory_sources', default=None, editable=False, to='main.Inventory', null=True),
+            field=models.ForeignKey(related_name='inventory_sources', on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to='main.Inventory', null=True),
         ),
         migrations.AddField(
             model_name='inventorysource',
@@ -916,7 +916,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='adhoccommandevent',
             name='ad_hoc_command',
-            field=models.ForeignKey(related_name='ad_hoc_command_events', editable=False, to='main.AdHocCommand'),
+            field=models.ForeignKey(related_name='ad_hoc_command_events', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.AdHocCommand'),
         ),
         migrations.AddField(
             model_name='adhoccommand',

awx/main/migrations/0002_squashed_v300_release.py

@@ -13,7 +13,6 @@ from django.conf import settings
 from django.utils.timezone import now
 
 import jsonfield.fields
-import jsonbfield.fields
 import taggit.managers
 
 
@@ -144,7 +143,7 @@ class Migration(migrations.Migration):
                 ('category', models.CharField(max_length=128)),
                 ('value', models.TextField(blank=True)),
                 ('value_type', models.CharField(max_length=12, choices=[('string', 'String'), ('int', 'Integer'), ('float', 'Decimal'), ('json', 'JSON'), ('bool', 'Boolean'), ('password', 'Password'), ('list', 'List')])),
-                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True)),
             ],
         ),
         # Notification changes
@@ -185,7 +184,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='notification',
             name='notification_template',
-            field=models.ForeignKey(related_name='notifications', editable=False, to='main.NotificationTemplate'),
+            field=models.ForeignKey(related_name='notifications', editable=False, on_delete=models.CASCADE, to='main.NotificationTemplate'),
         ),
         migrations.AddField(
             model_name='activitystream',
@@ -239,8 +238,8 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('timestamp', models.DateTimeField(default=None, help_text='Date and time of the corresponding fact scan gathering time.', editable=False)),
                 ('module', models.CharField(max_length=128)),
-                ('facts', jsonbfield.fields.JSONField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)),
-                ('host', models.ForeignKey(related_name='facts', to='main.Host', help_text='Host for the facts that the fact scan captured.')),
+                ('facts', awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)),
+                ('host', models.ForeignKey(related_name='facts', to='main.Host', on_delete=models.CASCADE, help_text='Host for the facts that the fact scan captured.')),
             ],
         ),
         migrations.AlterIndexTogether(
@@ -318,7 +317,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='project',
             name='organization',
-            field=models.ForeignKey(related_name='projects', to='main.Organization', blank=True, null=True),
+            field=models.ForeignKey(related_name='projects', to='main.Organization', on_delete=models.CASCADE, blank=True, null=True),
         ),
         migrations.AlterField(
             model_name='team',
@@ -367,7 +366,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='credential',
             name='organization',
-            field=models.ForeignKey(related_name='credentials', default=None, blank=True, to='main.Organization', null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=models.CASCADE, default=None, blank=True, to='main.Organization', null=True),
         ),
 
         #
@@ -382,7 +381,7 @@ class Migration(migrations.Migration):
                 ('members', models.ManyToManyField(related_name='roles', to=settings.AUTH_USER_MODEL)),
                 ('parents', models.ManyToManyField(related_name='children', to='main.Role')),
                 ('implicit_parents', models.TextField(default='[]')),
-                ('content_type', models.ForeignKey(default=None, to='contenttypes.ContentType', null=True)),
+                ('content_type', models.ForeignKey(default=None, to='contenttypes.ContentType', on_delete=models.CASCADE, null=True)),
                 ('object_id', models.PositiveIntegerField(default=None, null=True)),
 
             ],
@@ -398,8 +397,8 @@ class Migration(migrations.Migration):
                 ('role_field', models.TextField()),
                 ('content_type_id', models.PositiveIntegerField()),
                 ('object_id', models.PositiveIntegerField()),
-                ('ancestor', models.ForeignKey(related_name='+', to='main.Role')),
-                ('descendent', models.ForeignKey(related_name='+', to='main.Role')),
+                ('ancestor', models.ForeignKey(on_delete=models.CASCADE, related_name='+', to='main.Role')),
+                ('descendent', models.ForeignKey(on_delete=models.CASCADE, related_name='+', to='main.Role')),
             ],
             options={
                 'db_table': 'main_rbac_role_ancestors',
@@ -569,7 +568,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=512)),
                 ('created_by', models.ForeignKey(related_name="{u'class': 'label', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('modified_by', models.ForeignKey(related_name="{u'class': 'label', u'app_label': 'main'}(class)s_modified+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
-                ('organization', models.ForeignKey(related_name='labels', to='main.Organization', help_text='Organization this label belongs to.')),
+                ('organization', models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.CASCADE, to='main.Organization', help_text='Organization this label belongs to.')),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')),
             ],
             options={
@@ -599,12 +598,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='label',
             name='organization',
-            field=models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.Organization', help_text='Organization this label belongs to.', null=True),
+            field=models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.CASCADE, default=None, blank=True, to='main.Organization', help_text='Organization this label belongs to.', null=True),
         ),
         migrations.AlterField(
             model_name='label',
             name='organization',
-            field=models.ForeignKey(related_name='labels', to='main.Organization', help_text='Organization this label belongs to.'),
+            field=models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.CASCADE, to='main.Organization', help_text='Organization this label belongs to.'),
         ),
         # InventorySource Credential
         migrations.AddField(
@@ -630,12 +629,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='credential',
             name='deprecated_team',
-            field=models.ForeignKey(related_name='deprecated_credentials', default=None, blank=True, to='main.Team', null=True),
+            field=models.ForeignKey(related_name='deprecated_credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.Team', null=True),
         ),
         migrations.AlterField(
             model_name='credential',
             name='deprecated_user',
-            field=models.ForeignKey(related_name='deprecated_credentials', default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
+            field=models.ForeignKey(related_name='deprecated_credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
         ),
         migrations.AlterField(
             model_name='credential',

awx/main/migrations/0003_squashed_v300_v303_updates.py

@@ -116,7 +116,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='team',
             name='organization',
-            field=models.ForeignKey(related_name='teams', to='main.Organization'),
+            field=models.ForeignKey(related_name='teams', on_delete=models.CASCADE, to='main.Organization'),
             preserve_default=False,
         ),
     ] + _squashed.operations(SQUASHED_30, applied=True)

awx/main/migrations/0004_squashed_v310_release.py

@@ -74,7 +74,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='WorkflowJob',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, on_delete=models.CASCADE, primary_key=True, serialize=False, to='main.UnifiedJob')),
                 ('extra_vars', models.TextField(default='', blank=True)),
             ],
             options={
@@ -100,7 +100,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='WorkflowJobTemplate',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=models.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('extra_vars', models.TextField(default='', blank=True)),
                 ('admin_role', awx.main.fields.ImplicitRoleField(related_name='+', parent_role='singleton:system_administrator', to='main.Role', null='True')),
             ],
@@ -116,7 +116,7 @@ class Migration(migrations.Migration):
                 ('failure_nodes', models.ManyToManyField(related_name='workflowjobtemplatenodes_failure', to='main.WorkflowJobTemplateNode', blank=True)),
                 ('success_nodes', models.ManyToManyField(related_name='workflowjobtemplatenodes_success', to='main.WorkflowJobTemplateNode', blank=True)),
                 ('unified_job_template', models.ForeignKey(related_name='workflowjobtemplatenodes', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.UnifiedJobTemplate', null=True)),
-                ('workflow_job_template', models.ForeignKey(related_name='workflow_job_template_nodes', default=None, blank=True, to='main.WorkflowJobTemplate', null=True)),
+                ('workflow_job_template', models.ForeignKey(related_name='workflow_job_template_nodes', on_delete=models.SET_NULL, default=None, blank=True, to='main.WorkflowJobTemplate', null=True)),
             ],
             options={
                 'abstract': False,
@@ -161,7 +161,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobnode',
             name='char_prompts',
-            field=jsonfield.fields.JSONField(default={}, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
@@ -191,7 +191,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='char_prompts',
-            field=jsonfield.fields.JSONField(default={}, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
@@ -211,7 +211,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='workflowjobnode',
             name='workflow_job',
-            field=models.ForeignKey(related_name='workflow_job_nodes', default=None, blank=True, to='main.WorkflowJob', null=True),
+            field=models.ForeignKey(related_name='workflow_job_nodes', on_delete=django.db.models.deletion.CASCADE, default=None, blank=True, to='main.WorkflowJob', null=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplate',
@@ -227,12 +227,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='job',
             name='artifacts',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
             name='ancestor_artifacts',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         # Job timeout settings
         migrations.AddField(
@@ -397,7 +397,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjob',
             name='survey_passwords',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplate',
@@ -407,33 +407,33 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='survey_spec',
-            field=jsonfield.fields.JSONField(default={}, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, blank=True),
         ),
         # JSON field changes
         migrations.AlterField(
             model_name='adhoccommandevent',
             name='event_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='job',
             name='artifacts',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='job',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='jobevent',
             name='event_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='survey_spec',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='notification',
@@ -453,37 +453,37 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='schedule',
             name='extra_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='unifiedjob',
             name='job_env',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjob',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobnode',
             name='ancestor_artifacts',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobnode',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplate',
             name='survey_spec',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplatenode',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         # Job Project Update
         migrations.AddField(

awx/main/migrations/0006_v320_release.py

@@ -55,12 +55,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='inventorysource',
             name='deprecated_group',
-            field=models.OneToOneField(related_name='deprecated_inventory_source', null=True, default=None, to='main.Group'),
+            field=models.OneToOneField(related_name='deprecated_inventory_source', on_delete=models.CASCADE, null=True, default=None, to='main.Group'),
         ),
         migrations.AlterField(
             model_name='inventorysource',
             name='inventory',
-            field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', null=True),
+            field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', on_delete=models.CASCADE, null=True),
         ),
 
         # Smart Inventory
@@ -78,13 +78,13 @@ class Migration(migrations.Migration):
             name='SmartInventoryMembership',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('host', models.ForeignKey(related_name='+', to='main.Host')),
+                ('host', models.ForeignKey(related_name='+', on_delete=models.CASCADE, to='main.Host')),
             ],
         ),
         migrations.AddField(
             model_name='smartinventorymembership',
             name='inventory',
-            field=models.ForeignKey(related_name='+', to='main.Inventory'),
+            field=models.ForeignKey(on_delete=models.CASCADE, related_name='+', to='main.Inventory'),
         ),
         migrations.AddField(
             model_name='host',
@@ -105,19 +105,19 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='inventory',
             name='organization',
-            field=models.ForeignKey(related_name='inventories', on_delete=models.deletion.SET_NULL, to='main.Organization', help_text='Organization containing this inventory.', null=True),
+            field=models.ForeignKey(related_name='inventories', on_delete=models.SET_NULL, to='main.Organization', help_text='Organization containing this inventory.', null=True),
         ),
 
         # Facts
         migrations.AlterField(
             model_name='fact',
             name='facts',
-            field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
+            field=awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
         ),
         migrations.AddField(
             model_name='host',
             name='ansible_facts',
-            field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
+            field=awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
         ),
         migrations.AddField(
             model_name='host',
@@ -148,12 +148,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventorysource',
             name='source_project',
-            field=models.ForeignKey(related_name='scm_inventory_sources', default=None, blank=True, to='main.Project', help_text='Project containing inventory file used as source.', null=True),
+            field=models.ForeignKey(related_name='scm_inventory_sources', on_delete=models.CASCADE, default=None, blank=True, to='main.Project', help_text='Project containing inventory file used as source.', null=True),
         ),
         migrations.AddField(
             model_name='inventoryupdate',
             name='source_project_update',
-            field=models.ForeignKey(related_name='scm_inventory_updates', default=None, blank=True, to='main.ProjectUpdate', help_text='Inventory files from this Project Update were used for the inventory update.', null=True),
+            field=models.ForeignKey(related_name='scm_inventory_updates', on_delete=models.CASCADE, default=None, blank=True, to='main.ProjectUpdate', help_text='Inventory files from this Project Update were used for the inventory update.', null=True),
         ),
         migrations.AddField(
             model_name='project',
@@ -200,7 +200,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='notificationtemplate',
             name='organization',
-            field=models.ForeignKey(related_name='notification_templates', to='main.Organization', null=True),
+            field=models.ForeignKey(related_name='notification_templates', on_delete=models.CASCADE, to='main.Organization', null=True),
         ),
         migrations.AlterUniqueTogether(
             name='notificationtemplate',
@@ -312,7 +312,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventory',
             name='insights_credential',
-            field=models.ForeignKey(related_name='insights_inventories', on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.Credential', help_text='Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.', null=True),
+            field=models.ForeignKey(related_name='insights_inventories', on_delete=models.SET_NULL, default=None, blank=True, to='main.Credential', help_text='Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.', null=True),
         ),
         migrations.AlterField(
             model_name='inventory',
@@ -382,10 +382,10 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=512)),
                 ('kind', models.CharField(max_length=32, choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('insights', 'Insights')])),
                 ('managed_by_tower', models.BooleanField(default=False, editable=False)),
-                ('inputs', awx.main.fields.CredentialTypeInputField(default={}, blank=True, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
-                ('injectors', awx.main.fields.CredentialTypeInjectorField(default={}, blank=True, help_text='Enter injectors using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
-                ('created_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_created+", on_delete=models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
-                ('modified_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_modified+", on_delete=models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('inputs', awx.main.fields.CredentialTypeInputField(default=dict, blank=True, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
+                ('injectors', awx.main.fields.CredentialTypeInjectorField(default=dict, blank=True, help_text='Enter injectors using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
+                ('created_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_created+", on_delete=models.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('modified_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_modified+", on_delete=models.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')),
             ],
             options={
@@ -399,23 +399,23 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='credential',
             name='inputs',
-            field=awx.main.fields.CredentialInputField(default={}, blank=True),
+            field=awx.main.fields.CredentialInputField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='credential',
             name='credential_type',
-            field=models.ForeignKey(related_name='credentials', to='main.CredentialType', null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=models.CASCADE, to='main.CredentialType', null=True),
             preserve_default=False,
         ),
         migrations.AddField(
             model_name='job',
             name='vault_credential',
-            field=models.ForeignKey(related_name='jobs_as_vault_credential+', on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
+            field=models.ForeignKey(related_name='jobs_as_vault_credential+', on_delete=models.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
         ),
         migrations.AddField(
             model_name='jobtemplate',
             name='vault_credential',
-            field=models.ForeignKey(related_name='jobtemplates_as_vault_credential+', on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
+            field=models.ForeignKey(related_name='jobtemplates_as_vault_credential+', on_delete=models.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
         ),
         migrations.AddField(
             model_name='job',
@@ -452,7 +452,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(unique=True, max_length=250)),
                 ('created', models.DateTimeField(auto_now_add=True)),
                 ('modified', models.DateTimeField(auto_now=True)),
-                ('controller', models.ForeignKey(related_name='controlled_groups', default=None, editable=False, to='main.InstanceGroup', help_text='Instance Group to remotely control this group.', null=True)),
+                ('controller', models.ForeignKey(related_name='controlled_groups', on_delete=models.CASCADE, default=None, editable=False, to='main.InstanceGroup', help_text='Instance Group to remotely control this group.', null=True)),
                 ('instances', models.ManyToManyField(help_text='Instances that are members of this InstanceGroup', related_name='rampart_groups', editable=False, to='main.Instance')),
             ],
         ),
@@ -464,7 +464,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjob',
             name='instance_group',
-            field=models.ForeignKey(on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.InstanceGroup', help_text='The Rampart/Instance group the job was run under', null=True),
+            field=models.ForeignKey(on_delete=models.SET_NULL, default=None, blank=True, to='main.InstanceGroup', help_text='The Rampart/Instance group the job was run under', null=True),
         ),
         migrations.AddField(
             model_name='unifiedjobtemplate',

awx/main/migrations/0008_v320_drop_v1_credential_fields.py

@@ -103,12 +103,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='credential',
             name='credential_type',
-            field=models.ForeignKey(related_name='credentials', to='main.CredentialType', null=False, help_text='Specify the type of credential you want to create. Refer to the Ansible Tower documentation for details on each type.')
+            field=models.ForeignKey(related_name='credentials', to='main.CredentialType', on_delete=models.CASCADE, null=False, help_text='Specify the type of credential you want to create. Refer to the Ansible Tower documentation for details on each type.')
         ),
         migrations.AlterField(
             model_name='credential',
             name='inputs',
-            field=awx.main.fields.CredentialInputField(default={}, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.', blank=True),
+            field=awx.main.fields.CredentialInputField(default=dict, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.', blank=True),
         ),
         migrations.RemoveField(
             model_name='job',

awx/main/migrations/0014_v330_saved_launchtime_configs.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='schedule',
@@ -35,7 +35,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
@@ -45,12 +45,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobnode',
             name='extra_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
@@ -60,12 +60,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='extra_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         # Run data migration before removing the old credential field
         migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrations.RunPython.noop),
@@ -83,9 +83,9 @@ class Migration(migrations.Migration):
             name='JobLaunchConfig',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('extra_data', awx.main.fields.JSONField(blank=True, default={})),
-                ('survey_passwords', awx.main.fields.JSONField(blank=True, default={}, editable=False)),
-                ('char_prompts', awx.main.fields.JSONField(blank=True, default={})),
+                ('extra_data', awx.main.fields.JSONField(blank=True, default=dict)),
+                ('survey_passwords', awx.main.fields.JSONField(blank=True, default=dict, editable=False)),
+                ('char_prompts', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('credentials', models.ManyToManyField(related_name='joblaunchconfigs', to='main.Credential')),
                 ('inventory', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='joblaunchconfigs', to='main.Inventory')),
                 ('job', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='launch_config', to='main.UnifiedJob')),
@@ -94,51 +94,51 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='ask_variables_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_credential_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_diff_mode_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_inventory_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_job_type_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_limit_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_skip_tags_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_tags_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_variables_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_verbosity_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
     ]

awx/main/migrations/0018_v330_add_additional_stdout_events.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default={})),
+                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
                 ('counter', models.PositiveIntegerField(default=0, editable=False)),
                 ('stdout', models.TextField(default='', editable=False)),
@@ -40,7 +40,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('event', models.CharField(choices=[('runner_on_failed', 'Host Failed'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_item_on_ok', 'Item OK'), ('runner_item_on_failed', 'Item Failed'), ('runner_item_on_skipped', 'Item Skipped'), ('runner_retry', 'Host Retry'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_include', 'Including File'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete'), ('debug', 'Debug'), ('verbose', 'Verbose'), ('deprecated', 'Deprecated'), ('warning', 'Warning'), ('system_warning', 'System Warning'), ('error', 'Error')], max_length=100)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default={})),
+                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
@@ -65,7 +65,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default={})),
+                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
                 ('counter', models.PositiveIntegerField(default=0, editable=False)),
                 ('stdout', models.TextField(default='', editable=False)),

awx/main/migrations/0053_v340_workflow_inventory.py

@@ -17,7 +17,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjob',
             name='char_prompts',
-            field=awx.main.fields.JSONField(blank=True, default={}),
+            field=awx.main.fields.JSONField(blank=True, default=dict),
         ),
         migrations.AddField(
             model_name='workflowjob',
@@ -27,7 +27,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='ask_inventory_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AddField(
             model_name='workflowjobtemplate',

awx/main/migrations/0067_v350_credential_plugins.py

@@ -34,7 +34,7 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('description', models.TextField(blank=True, default='')),
                 ('input_field_name', models.CharField(max_length=1024)),
-                ('metadata', awx.main.fields.DynamicCredentialInputField(blank=True, default={})),
+                ('metadata', awx.main.fields.DynamicCredentialInputField(blank=True, default=dict)),
                 ('created_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_created+", to=settings.AUTH_USER_MODEL)),
                 ('modified_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_modified+", to=settings.AUTH_USER_MODEL)),
                 ('source_credential', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='target_input_sources', to='main.Credential')),

awx/main/migrations/0081_v360_notify_on_start.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-30 20:35
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+def forwards_split_unified_job_template_any(apps, schema_editor):
+    UnifiedJobTemplate = apps.get_model('main', 'unifiedjobtemplate')
+    for ujt in UnifiedJobTemplate.objects.all():
+        for ujt_notification in ujt.notification_templates_any.all():
+            ujt.notification_templates_success.add(ujt_notification)
+            ujt.notification_templates_error.add(ujt_notification)
+
+def forwards_split_organization_any(apps, schema_editor):
+    Organization = apps.get_model('main', 'organization')
+    for org in Organization.objects.all():
+        for org_notification in org.notification_templates_any.all():
+            org.notification_templates_success.add(org_notification)
+            org.notification_templates_error.add(org_notification)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0080_v360_replace_job_origin'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='organization',
+            name='notification_templates_started',
+            field=models.ManyToManyField(blank=True, related_name='organization_notification_templates_for_started', to='main.NotificationTemplate'),
+        ),
+        migrations.AddField(
+            model_name='unifiedjobtemplate',
+            name='notification_templates_started',
+            field=models.ManyToManyField(blank=True, related_name='unifiedjobtemplate_notification_templates_for_started', to='main.NotificationTemplate'),
+        ),
+        # Separate out "any" notifications into "success" and "error" before the "any" state gets deleted.
+        migrations.RunPython(forwards_split_unified_job_template_any, None),
+        migrations.RunPython(forwards_split_organization_any, None),
+        migrations.RemoveField(
+            model_name='organization',
+            name='notification_templates_any',
+        ),
+        migrations.RemoveField(
+            model_name='unifiedjobtemplate',
+            name='notification_templates_any',
+        ),
+    ]

awx/main/migrations/0082_v360_webhook_http_method.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def add_webhook_notification_template_fields(apps, schema_editor):
+    # loop over all existing webhook notification templates and make
+    # sure they have the new "http_method" field filled in with "POST"
+    NotificationTemplate = apps.get_model('main', 'notificationtemplate')
+    webhooks = NotificationTemplate.objects.filter(notification_type='webhook')
+    for w in webhooks:
+        w.notification_configuration['http_method'] = 'POST'
+        w.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0081_v360_notify_on_start'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_webhook_notification_template_fields, migrations.RunPython.noop),
+    ]

awx/main/migrations/0083_v360_job_branch_override.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-14 15:08
+from __future__ import unicode_literals
+
+import awx.main.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0082_v360_webhook_http_method'),
+    ]
+
+    operations = [
+        # Add fields for user-provided project refspec
+        migrations.AddField(
+            model_name='project',
+            name='scm_refspec',
+            field=models.CharField(blank=True, default='', help_text='For git projects, an additional refspec to fetch.', max_length=1024, verbose_name='SCM refspec'),
+        ),
+        migrations.AddField(
+            model_name='projectupdate',
+            name='scm_refspec',
+            field=models.CharField(blank=True, default='', help_text='For git projects, an additional refspec to fetch.', max_length=1024, verbose_name='SCM refspec'),
+        ),
+        # Add fields for job specification of project branch
+        migrations.AddField(
+            model_name='job',
+            name='scm_branch',
+            field=models.CharField(blank=True, default='', help_text='Branch to use in job run. Project default used if blank. Only allowed if project allow_override field is set to true.', max_length=1024),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='ask_scm_branch_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='scm_branch',
+            field=models.CharField(blank=True, default='', help_text='Branch to use in job run. Project default used if blank. Only allowed if project allow_override field is set to true.', max_length=1024),
+        ),
+        migrations.AddField(
+            model_name='project',
+            name='allow_override',
+            field=models.BooleanField(default=False, help_text='Allow changing the SCM branch or revision in a job template that uses this project.'),
+        ),
+        # Fix typo in help_text
+        migrations.AlterField(
+            model_name='project',
+            name='scm_update_cache_timeout',
+            field=models.PositiveIntegerField(blank=True, default=0, help_text='The number of seconds after the last project update ran that a new project update will be launched as a job dependency.'),
+        ),
+        # Start tracking the fetched revision on project update model
+        migrations.AddField(
+            model_name='projectupdate',
+            name='scm_revision',
+            field=models.CharField(blank=True, default='', editable=False, help_text='The SCM Revision discovered by this update for the given project and branch.', max_length=1024, verbose_name='SCM Revision'),
+        ),
+    ]

awx/main/migrations/0084_v360_token_description.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.4 on 2019-08-16 13:22
+
+from django.db import migrations, models
+
+import awx
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0083_v360_job_branch_override'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='oauth2accesstoken',
+            name='description',
+            field=models.TextField(blank=True, default=''),
+        ),
+    ]

awx/main/migrations/0085_v360_add_notificationtemplate_messages.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-10 16:56
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+import awx.main.fields
+import awx.main.models.notifications
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0084_v360_token_description'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='notificationtemplate',
+            name='messages',
+            field=awx.main.fields.JSONField(default=awx.main.models.notifications.NotificationTemplate.default_messages,
+                                            help_text='Optional custom messages for notification template.',
+                                            null=True,
+                                            blank=True),
+        ),
+        migrations.AlterField(
+            model_name='notification',
+            name='notification_type',
+            field=models.CharField(choices=[('email', 'Email'), ('grafana', 'Grafana'), ('hipchat', 'HipChat'), ('irc', 'IRC'), ('mattermost', 'Mattermost'), ('pagerduty', 'Pagerduty'), ('rocketchat', 'Rocket.Chat'), ('slack', 'Slack'), ('twilio', 'Twilio'), ('webhook', 'Webhook')], max_length=32),
+        ),
+        migrations.AlterField(
+            model_name='notificationtemplate',
+            name='notification_type',
+            field=models.CharField(choices=[('email', 'Email'), ('grafana', 'Grafana'), ('hipchat', 'HipChat'), ('irc', 'IRC'), ('mattermost', 'Mattermost'), ('pagerduty', 'Pagerduty'), ('rocketchat', 'Rocket.Chat'), ('slack', 'Slack'), ('twilio', 'Twilio'), ('webhook', 'Webhook')], max_length=32),
+        ),
+    ]

awx/main/migrations/0086_v360_workflow_approval.py

@@ -0,0 +1,83 @@
+# Generated by Django 2.2.4 on 2019-08-02 17:51
+
+import awx.main.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0085_v360_add_notificationtemplate_messages'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='WorkflowApprovalTemplate',
+            fields=[
+                ('unifiedjobtemplate_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('timeout', models.IntegerField(blank=True, default=0, help_text='The amount of time (in seconds) before the approval node expires and fails.')),
+            ],
+            bases=('main.unifiedjobtemplate',),
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='approval_role',
+            field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role='admin_role', related_name='+', to='main.Role'),
+            preserve_default='True',
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='approval_role',
+            field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role=['organization.approval_role', 'admin_role'], related_name='+', to='main.Role'),
+            preserve_default='True',
+        ),
+        migrations.AlterField(
+            model_name='workflowjobnode',
+            name='unified_job_template',
+            field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='workflowjobnodes', to='main.UnifiedJobTemplate'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplatenode',
+            name='unified_job_template',
+            field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='workflowjobtemplatenodes', to='main.UnifiedJobTemplate'),
+        ),
+        migrations.CreateModel(
+            name='WorkflowApproval',
+            fields=[
+                ('unifiedjob_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('workflow_approval_template', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='approvals', to='main.WorkflowApprovalTemplate')),
+            ],
+            bases=('main.unifiedjob',),
+        ),
+        migrations.AddField(
+            model_name='activitystream',
+            name='workflow_approval',
+            field=models.ManyToManyField(blank=True, to='main.WorkflowApproval'),
+        ),
+        migrations.AddField(
+            model_name='activitystream',
+            name='workflow_approval_template',
+            field=models.ManyToManyField(blank=True, to='main.WorkflowApprovalTemplate'),
+        ),
+        migrations.AlterField(
+            model_name='organization',
+            name='read_role',
+            field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role=['member_role', 'auditor_role', 'execute_role', 'project_admin_role', 'inventory_admin_role', 'workflow_admin_role', 'notification_admin_role', 'credential_admin_role', 'job_template_admin_role', 'approval_role'], related_name='+', to='main.Role'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplate',
+            name='read_role',
+            field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role=['singleton:system_auditor', 'organization.auditor_role', 'execute_role', 'admin_role', 'approval_role'], related_name='+', to='main.Role'),
+        ),
+        migrations.AddField(
+            model_name='workflowapproval',
+            name='timeout',
+            field=models.IntegerField(blank=True, default=0, help_text='The amount of time (in seconds) before the approval node expires and fails.'),
+        ),
+        migrations.AddField(
+            model_name='workflowapproval',
+            name='timed_out',
+            field=models.BooleanField(default=False, help_text='Shows when an approval node (with a timeout assigned to it) has timed out.'),
+        ),
+    ]

awx/main/migrations/0087_v360_update_credential_injector_help_text.py

@@ -0,0 +1,29 @@
+# Generated by Django 2.2.4 on 2019-08-27 21:50
+
+import awx.main.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0086_v360_workflow_approval'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='credential',
+            name='inputs',
+            field=awx.main.fields.CredentialInputField(blank=True, default=dict, help_text='Enter inputs using either JSON or YAML syntax. Refer to the Ansible Tower documentation for example syntax.'),
+        ),
+        migrations.AlterField(
+            model_name='credentialtype',
+            name='injectors',
+            field=awx.main.fields.CredentialTypeInjectorField(blank=True, default=dict, help_text='Enter injectors using either JSON or YAML syntax. Refer to the Ansible Tower documentation for example syntax.'),
+        ),
+        migrations.AlterField(
+            model_name='credentialtype',
+            name='inputs',
+            field=awx.main.fields.CredentialTypeInputField(blank=True, default=dict, help_text='Enter inputs using either JSON or YAML syntax. Refer to the Ansible Tower documentation for example syntax.'),
+        ),
+    ]

awx/main/migrations/_rbac.py

@@ -1,17 +1,11 @@
 import logging
 from time import time
 
-from django.utils.encoding import smart_text
-from django.db import transaction
-from django.db.models import Q
-from django.db.utils import IntegrityError
-
-from collections import defaultdict
-from awx.main.utils import getattrd
 from awx.main.models.rbac import Role, batch_role_ancestor_rebuilding
 
 logger = logging.getLogger('rbac_migrations')
 
+
 def create_roles(apps, schema_editor):
     '''
     Implicit role creation happens in our post_save hook for all of our
@@ -41,465 +35,19 @@ def create_roles(apps, schema_editor):
                 obj.save()
 
 
-def migrate_users(apps, schema_editor):
-    User = apps.get_model('auth', "User")
-    Role = apps.get_model('main', "Role")
-    ContentType = apps.get_model('contenttypes', "ContentType")
-    user_content_type = ContentType.objects.get_for_model(User)
-
-    for user in User.objects.iterator():
-        user.save()
-        try:
-            Role.objects.get(content_type=user_content_type, object_id=user.id)
-            logger.info(smart_text(u"found existing role for user: {}".format(user.username)))
-        except Role.DoesNotExist:
-            role = Role.objects.create(
-                role_field='admin_role',
-                content_type = user_content_type,
-                object_id = user.id
-            )
-            role.members.add(user)
-            logger.info(smart_text(u"migrating to new role for user: {}".format(user.username)))
-
-        if user.is_superuser:
-            if Role.objects.filter(singleton_name='system_administrator').exists():
-                sa_role = Role.objects.get(singleton_name='system_administrator')
-            else:
-                sa_role = Role.objects.create(
-                    singleton_name='system_administrator',
-                    role_field='system_administrator'
-                )
-
-            sa_role.members.add(user)
-            logger.warning(smart_text(u"added superuser: {}".format(user.username)))
-
-def migrate_organization(apps, schema_editor):
-    Organization = apps.get_model('main', "Organization")
-    for org in Organization.objects.iterator():
-        for admin in org.deprecated_admins.all():
-            org.admin_role.members.add(admin)
-            logger.info(smart_text(u"added admin: {}, {}".format(org.name, admin.username)))
-        for user in org.deprecated_users.all():
-            org.member_role.members.add(user)
-            logger.info(smart_text(u"added member: {}, {}".format(org.name, user.username)))
-
-def migrate_team(apps, schema_editor):
-    Team = apps.get_model('main', 'Team')
-    for t in Team.objects.iterator():
-        for user in t.deprecated_users.all():
-            t.member_role.members.add(user)
-            logger.info(smart_text(u"team: {}, added user: {}".format(t.name, user.username)))
-
-def attrfunc(attr_path):
-    '''attrfunc returns a function that will
-    attempt to use the attr_path to access the attribute
-    of an instance that is passed in to the returned function.
-
-    Example:
-        get_org = attrfunc('inventory.organization')
-        org = get_org(JobTemplateInstance)
-    '''
-    def attr(inst):
-        return getattrd(inst, attr_path)
-    return attr
-
-def _update_credential_parents(org, cred):
-    cred.organization = org
-    cred.save()
-
-def _discover_credentials(instances, cred, orgfunc):
-    '''_discover_credentials will find shared credentials across
-    organizations. If a shared credential is found, it will duplicate
-    the credential, ensure the proper role permissions are added to the new
-    credential, and update any references from the old to the newly created
-    credential.
-
-    instances is a list of all objects that were matched when filtered
-    with cred.
-
-    orgfunc is a function that when called with an instance from instances
-    will produce an Organization object.
-    '''
-    orgs = defaultdict(list)
-    for inst in instances:
-        try:
-            orgs[orgfunc(inst)].append(inst)
-        except AttributeError:
-            # JobTemplate.inventory can be NULL sometimes, eg when an inventory
-            # has been deleted. This protects against that.
-            pass
-
-    if len(orgs) == 1:
-        try:
-            _update_credential_parents(orgfunc(instances[0]), cred)
-        except AttributeError:
-            # JobTemplate.inventory can be NULL sometimes, eg when an inventory
-            # has been deleted. This protects against that.
-            pass
-    else:
-        for pos, org in enumerate(orgs):
-            if pos == 0:
-                _update_credential_parents(org, cred)
-            else:
-                # Create a new credential
-                cred.pk = None
-                cred.organization = None
-                cred.save()
-
-                cred.admin_role, cred.use_role = None, None
-
-                for i in orgs[org]:
-                    i.credential = cred
-                    i.save()
-
-                _update_credential_parents(org, cred)
-
-def migrate_credential(apps, schema_editor):
-    Credential = apps.get_model('main', "Credential")
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-    Project = apps.get_model('main', 'Project')
-    InventorySource = apps.get_model('main', 'InventorySource')
-
-    for cred in Credential.objects.iterator():
-        results = [x for x in JobTemplate.objects.filter(Q(credential=cred) | Q(cloud_credential=cred), inventory__isnull=False).all()] + \
-                  [x for x in InventorySource.objects.filter(credential=cred).all()]
-        if cred.deprecated_team is not None and results:
-            if len(results) == 1:
-                _update_credential_parents(results[0].inventory.organization, cred)
-            else:
-                _discover_credentials(results, cred, attrfunc('inventory.organization'))
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at organization level".format(cred.name, cred.kind, cred.host)))
-
-        projs = Project.objects.filter(credential=cred).all()
-        if cred.deprecated_team is not None and projs:
-            if len(projs) == 1:
-                _update_credential_parents(projs[0].organization, cred)
-            else:
-                _discover_credentials(projs, cred, attrfunc('organization'))
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at organization level".format(cred.name, cred.kind, cred.host)))
-
-        if cred.deprecated_team is not None:
-            cred.deprecated_team.admin_role.children.add(cred.admin_role)
-            cred.deprecated_team.member_role.children.add(cred.use_role)
-            cred.save()
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at user level".format(cred.name, cred.kind, cred.host)))
-        elif cred.deprecated_user is not None:
-            cred.admin_role.members.add(cred.deprecated_user)
-            cred.save()
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at user level".format(cred.name, cred.kind, cred.host, )))
-        else:
-            logger.warning(smart_text(u"orphaned credential found Credential(name={}, kind={}, host={}), superuser only".format(cred.name, cred.kind, cred.host, )))
-
-
-def migrate_inventory(apps, schema_editor):
-    Inventory = apps.get_model('main', 'Inventory')
-    Permission = apps.get_model('main', 'Permission')
-
-    def role_from_permission(perm):
-        if perm.permission_type == 'admin':
-            return inventory.admin_role
-        elif perm.permission_type == 'read':
-            return inventory.read_role
-        elif perm.permission_type == 'write':
-            return inventory.update_role
-        elif perm.permission_type == 'check' or perm.permission_type == 'run' or perm.permission_type == 'create':
-            # These permission types are handled differntly in RBAC now, nothing to migrate.
-            return False
-        else:
-            return None
-
-    for inventory in Inventory.objects.iterator():
-        for perm in Permission.objects.filter(inventory=inventory):
-            role = None
-            execrole = None
-
-            role = role_from_permission(perm)
-            if role is None:
-                raise Exception(smart_text(u'Unhandled permission type for inventory: {}'.format( perm.permission_type)))
-
-            if perm.run_ad_hoc_commands:
-                execrole = inventory.use_role
-
-            if perm.team:
-                if role:
-                    perm.team.member_role.children.add(role)
-                if execrole:
-                    perm.team.member_role.children.add(execrole)
-                logger.info(smart_text(u'added Team({}) access to Inventory({})'.format(perm.team.name, inventory.name)))
-
-            if perm.user:
-                if role:
-                    role.members.add(perm.user)
-                if execrole:
-                    execrole.members.add(perm.user)
-                logger.info(smart_text(u'added User({}) access to Inventory({})'.format(perm.user.username, inventory.name)))
-
-def migrate_projects(apps, schema_editor):
-    '''
-    I can see projects when:
-     X I am a superuser.
-     X I am an admin in an organization associated with the project.
-     X I am a user in an organization associated with the project.
-     X I am on a team associated with the project.
-     X I have been explicitly granted permission to run/check jobs using the
-       project.
-     X I created the project but it isn't associated with an organization
-    I can change/delete when:
-     X I am a superuser.
-     X I am an admin in an organization associated with the project.
-     X I created the project but it isn't associated with an organization
-    '''
-    Project = apps.get_model('main', 'Project')
-    Permission = apps.get_model('main', 'Permission')
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-
-    # Migrate projects to single organizations, duplicating as necessary
-    for project in Project.objects.iterator():
-        original_project_name = project.name
-        project_orgs = project.deprecated_organizations.distinct().all()
-
-        if len(project_orgs) >= 1:
-            first_org = None
-            for org in project_orgs:
-                if first_org is None:
-                    # For the first org, re-use our existing Project object, so don't do the below duplication effort
-                    first_org = org
-                    if len(project_orgs) > 1:
-                        project.name = smart_text(u'{} - {}'.format(first_org.name, original_project_name))
-                    project.organization = first_org
-                    project.save()
-                else:
-                    new_prj = Project.objects.create(
-                        created                   = project.created,
-                        modified                  = project.modified,
-                        polymorphic_ctype_id      = project.polymorphic_ctype_id,
-                        description               = project.description,
-                        name                      = smart_text(u'{} - {}'.format(org.name, original_project_name)),
-                        old_pk                    = project.old_pk,
-                        created_by_id             = project.created_by_id,
-                        modified_by_id            = project.modified_by_id,
-                        scm_type                  = project.scm_type,
-                        scm_url                   = project.scm_url,
-                        scm_branch                = project.scm_branch,
-                        scm_clean                 = project.scm_clean,
-                        scm_delete_on_update      = project.scm_delete_on_update,
-                        scm_delete_on_next_update = project.scm_delete_on_next_update,
-                        scm_update_on_launch      = project.scm_update_on_launch,
-                        scm_update_cache_timeout  = project.scm_update_cache_timeout,
-                        credential                = project.credential,
-                        organization              = org
-                    )
-                    if project.scm_type == "":
-                        new_prj.local_path = project.local_path
-                        new_prj.save()
-                    for team in project.deprecated_teams.iterator():
-                        new_prj.deprecated_teams.add(team)
-                    logger.warning(smart_text(u'cloning Project({}) onto {} as Project({})'.format(original_project_name, org, new_prj)))
-                    job_templates = JobTemplate.objects.filter(project=project, inventory__organization=org).all()
-                    for jt in job_templates:
-                        jt.project = new_prj
-                        jt.save()
-                    for perm in Permission.objects.filter(project=project):
-                        Permission.objects.create(
-                            created                   = perm.created,
-                            modified                  = perm.modified,
-                            created_by                = perm.created_by,
-                            modified_by               = perm.modified_by,
-                            description               = perm.description,
-                            name                      = perm.name,
-                            user                      = perm.user,
-                            team                      = perm.team,
-                            project                   = new_prj,
-                            inventory                 = perm.inventory,
-                            permission_type           = perm.permission_type,
-                            run_ad_hoc_commands       = perm.run_ad_hoc_commands,
-                        )
-
-    # Migrate permissions
-    for project in Project.objects.iterator():
-        if project.organization is None and project.created_by is not None:
-            project.admin_role.members.add(project.created_by)
-            logger.warn(smart_text(u'adding Project({}) admin: {}'.format(project.name, project.created_by.username)))
-
-        for team in project.deprecated_teams.all():
-            team.member_role.children.add(project.read_role)
-            logger.info(smart_text(u'adding Team({}) access for Project({})'.format(team.name, project.name)))
-
-        for perm in Permission.objects.filter(project=project):
-            if perm.permission_type == 'create':
-                role = project.use_role
-            else:
-                role = project.read_role
-
-            if perm.team:
-                perm.team.member_role.children.add(role)
-                logger.info(smart_text(u'adding Team({}) access for Project({})'.format(perm.team.name, project.name)))
-
-            if perm.user:
-                role.members.add(perm.user)
-                logger.info(smart_text(u'adding User({}) access for Project({})'.format(perm.user.username, project.name)))
-
-        if project.organization is not None:
-            for user in project.organization.deprecated_users.all():
-                if not (project.use_role.members.filter(pk=user.id).exists() or project.admin_role.members.filter(pk=user.id).exists()):
-                    project.read_role.members.add(user)
-                    logger.info(smart_text(u'adding Organization({}) member access to Project({})'.format(project.organization.name, project.name)))
-
-
-
-def migrate_job_templates(apps, schema_editor):
-    '''
-    NOTE: This must be run after orgs, inventory, projects, credential, and
-    users have been migrated
-    '''
-
-
-    '''
-    I can see job templates when:
-     X I am a superuser.
-     - I can read the inventory, project and credential (which means I am an
-       org admin or member of a team with access to all of the above).
-     - I have permission explicitly granted to check/deploy with the inventory
-       and project.
-
-
-    #This does not mean I would be able to launch a job from the template or
-    #edit the template.
-     - access.py can_read for JobTemplate enforces that you can only
-       see it if you can launch it, so the above imply launch too
-    '''
-
-
-    '''
-    Tower administrators, organization administrators, and project
-    administrators, within a project under their purview, may create and modify
-    new job templates for that project.
-
-    When editing a job template, they may select among the inventory groups and
-    credentials in the organization for which they have usage permissions, or
-    they may leave either blank to be selected at runtime.
-
-    Additionally, they may specify one or more users/teams that have execution
-    permission for that job template, among the users/teams that are a member
-    of that project.
-
-    That execution permission is valid irrespective of any explicit permissions
-    the user has or has not been granted to the inventory group or credential
-    specified in the job template.
-
-    '''
-
-    User = apps.get_model('auth', 'User')
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-    Team = apps.get_model('main', 'Team')
-    Permission = apps.get_model('main', 'Permission')
-    Credential = apps.get_model('main', 'Credential')
-
-    jt_queryset = JobTemplate.objects.select_related('inventory', 'project', 'inventory__organization', 'execute_role')
-
-    for jt in jt_queryset.iterator():
-        if jt.inventory is None:
-            # If inventory is None, then only system admins and org admins can
-            # do anything with the JT in 2.4
-            continue
-
-        jt_permission_qs = Permission.objects.filter(
-            inventory=jt.inventory,
-            project=jt.project,
-        )
-
-        inventory_permission_qs = Permission.objects.filter(
-            inventory=jt.inventory,
-            project__isnull=True,
-        )
-
-        team_create_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['create'])
-            .values_list('team__id', flat=True)
-        )
-        team_run_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['check', 'run'] if jt.job_type == 'check' else ['run'])
-            .values_list('team__id', flat=True)
-        )
-        user_create_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['create'])
-            .values_list('user__id', flat=True)
-        )
-        user_run_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['check', 'run'] if jt.job_type == 'check' else ['run'])
-            .values_list('user__id', flat=True)
-        )
-
-        team_inv_permissions = defaultdict(set)
-        user_inv_permissions = defaultdict(set)
-
-        for user_id, team_id, inventory_id in inventory_permission_qs.values_list('user_id', 'team_id', 'inventory_id'):
-            if user_id:
-                user_inv_permissions[user_id].add(inventory_id)
-            if team_id:
-                team_inv_permissions[team_id].add(inventory_id)
-
-
-        for team in Team.objects.filter(id__in=team_create_permissions).iterator():
-            if jt.inventory.id in team_inv_permissions[team.id] and \
-                ((not jt.credential and not jt.cloud_credential) or
-                    Credential.objects.filter(deprecated_team=team, jobtemplates=jt).exists()):
-                team.member_role.children.add(jt.admin_role)
-                logger.info(smart_text(u'transfering admin access on JobTemplate({}) to Team({})'.format(jt.name, team.name)))
-        for team in Team.objects.filter(id__in=team_run_permissions).iterator():
-            if jt.inventory.id in team_inv_permissions[team.id] and \
-               ((not jt.credential and not jt.cloud_credential) or
-                    Credential.objects.filter(deprecated_team=team, jobtemplates=jt).exists()):
-                team.member_role.children.add(jt.execute_role)
-                logger.info(smart_text(u'transfering execute access on JobTemplate({}) to Team({})'.format(jt.name, team.name)))
-
-        for user in User.objects.filter(id__in=user_create_permissions).iterator():
-            cred = jt.credential or jt.cloud_credential
-            if (jt.inventory.id in user_inv_permissions[user.id] or
-                    any([jt.inventory.id in team_inv_permissions[team.id] for team in user.deprecated_teams.all()])) and \
-                    (not cred or cred.deprecated_user == user or
-                        (cred.deprecated_team and cred.deprecated_team.deprecated_users.filter(pk=user.id).exists())):
-                jt.admin_role.members.add(user)
-                logger.info(smart_text(u'transfering admin access on JobTemplate({}) to User({})'.format(jt.name, user.username)))
-        for user in User.objects.filter(id__in=user_run_permissions).iterator():
-            cred = jt.credential or jt.cloud_credential
-
-            if (jt.inventory.id in user_inv_permissions[user.id] or
-                    any([jt.inventory.id in team_inv_permissions[team.id] for team in user.deprecated_teams.all()])) and \
-                    (not cred or cred.deprecated_user == user or
-                        (cred.deprecated_team and cred.deprecated_team.deprecated_users.filter(pk=user.id).exists())):
-                jt.execute_role.members.add(user)
-                logger.info(smart_text(u'transfering execute access on JobTemplate({}) to User({})'.format(jt.name, user.username)))
-
-
-
 def rebuild_role_hierarchy(apps, schema_editor):
-        logger.info('Computing role roots..')
-        start = time()
-        roots = Role.objects \
-                    .all() \
-                    .values_list('id', flat=True)
-        stop = time()
-        logger.info('Found %d roots in %f seconds, rebuilding ancestry map' % (len(roots), stop - start))
-        start = time()
-        Role.rebuild_role_ancestor_list(roots, [])
-        stop = time()
-        logger.info('Rebuild completed in %f seconds' % (stop - start))
-        logger.info('Done.')
-
-
-def infer_credential_org_from_team(apps, schema_editor):
-    Credential = apps.get_model('main', "Credential")
-    for cred in Credential.objects.exclude(deprecated_team__isnull=True):
-        try:
-            with transaction.atomic():
-                _update_credential_parents(cred.deprecated_team.organization, cred)
-        except IntegrityError:
-            logger.info("Organization<{}> credential for old Team<{}> credential already created".format(cred.deprecated_team.organization.pk, cred.pk))
+    logger.info('Computing role roots..')
+    start = time()
+    roots = Role.objects \
+                .all() \
+                .values_list('id', flat=True)
+    stop = time()
+    logger.info('Found %d roots in %f seconds, rebuilding ancestry map' % (len(roots), stop - start))
+    start = time()
+    Role.rebuild_role_ancestor_list(roots, [])
+    stop = time()
+    logger.info('Rebuild completed in %f seconds' % (stop - start))
+    logger.info('Done.')
 
 
 def delete_all_user_roles(apps, schema_editor):

awx/main/migrations/_squashed_30.py

@@ -30,7 +30,7 @@ SQUASHED_30 = {
         migrations.AddField(
             model_name='job',
             name='survey_passwords',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
     ],
     '0031_v302_migrate_survey_passwords': [

awx/main/models/__init__.py

@@ -7,7 +7,8 @@ from django.db.models.signals import pre_delete  # noqa
 
 # AWX
 from awx.main.models.base import (  # noqa
-    BaseModel, PrimordialModel, prevent_search, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES
+    BaseModel, PrimordialModel, prevent_search, accepts_json,
+    CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES
 )
 from awx.main.models.unified_jobs import (  # noqa
     UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded
@@ -48,11 +49,14 @@ from awx.main.models.mixins import (  # noqa
     TaskManagerJobMixin, TaskManagerProjectUpdateMixin,
     TaskManagerUnifiedJobMixin,
 )
-from awx.main.models.notifications import Notification, NotificationTemplate # noqa
+from awx.main.models.notifications import (  # noqa
+    Notification, NotificationTemplate,
+    JobNotificationMixin
+)
 from awx.main.models.label import Label # noqa
 from awx.main.models.workflow import (  # noqa
     WorkflowJob, WorkflowJobNode, WorkflowJobOptions, WorkflowJobTemplate,
-    WorkflowJobTemplateNode,
+    WorkflowJobTemplateNode, WorkflowApproval, WorkflowApprovalTemplate,
 )
 from awx.main.models.channels import ChannelGroup # noqa
 from awx.api.versioning import reverse
@@ -62,24 +66,6 @@ from awx.main.models.oauth import ( # noqa
 from oauth2_provider.models import Grant, RefreshToken # noqa -- needed django-oauth-toolkit model migrations
 
 
-
-# Monkeypatch Django serializer to ignore django-taggit fields (which break
-# the dumpdata command; see https://github.com/alex/django-taggit/issues/155).
-from django.core.serializers.python import Serializer as _PythonSerializer
-_original_handle_m2m_field = _PythonSerializer.handle_m2m_field
-
-
-def _new_handle_m2m_field(self, obj, field):
-    try:
-        field.rel.through._meta
-    except AttributeError:
-        return
-    return _original_handle_m2m_field(self, obj, field)
-
-
-_PythonSerializer.handle_m2m_field = _new_handle_m2m_field
-
-
 # Add custom methods to User model for permissions checks.
 from django.contrib.auth.models import User  # noqa
 from awx.main.access import (  # noqa
@@ -140,25 +126,29 @@ def user_is_system_auditor(user):
 
 @user_is_system_auditor.setter
 def user_is_system_auditor(user, tf):
-    if user.id:
-        if tf:
-            role = Role.singleton('system_auditor')
-            # must check if member to not duplicate activity stream
-            if user not in role.members.all():
-                role.members.add(user)
-            user._is_system_auditor = True
-        else:
-            role = Role.singleton('system_auditor')
-            if user in role.members.all():
-                role.members.remove(user)
-            user._is_system_auditor = False
+    if not user.id:
+        # If the user doesn't have a primary key yet (i.e., this is the *first*
+        # time they've logged in, and we've just created the new User in this
+        # request), we need one to set up the system auditor role
+        user.save()
+    if tf:
+        role = Role.singleton('system_auditor')
+        # must check if member to not duplicate activity stream
+        if user not in role.members.all():
+            role.members.add(user)
+        user._is_system_auditor = True
+    else:
+        role = Role.singleton('system_auditor')
+        if user in role.members.all():
+            role.members.remove(user)
+        user._is_system_auditor = False
 
 
 User.add_to_class('is_system_auditor', user_is_system_auditor)
 
 
 def user_is_in_enterprise_category(user, category):
-    ret = (category,) in user.enterprise_auth.all().values_list('provider') and not user.has_usable_password()
+    ret = (category,) in user.enterprise_auth.values_list('provider') and not user.has_usable_password()
     # NOTE: this if-else block ensures existing enterprise users are still able to
     # log in. Remove it in a future release
     if category == 'radius':
@@ -213,6 +203,8 @@ activity_stream_registrar.connect(User)
 activity_stream_registrar.connect(WorkflowJobTemplate)
 activity_stream_registrar.connect(WorkflowJobTemplateNode)
 activity_stream_registrar.connect(WorkflowJob)
+activity_stream_registrar.connect(WorkflowApproval)
+activity_stream_registrar.connect(WorkflowApprovalTemplate)
 activity_stream_registrar.connect(OAuth2Application)
 activity_stream_registrar.connect(OAuth2AccessToken)
 
@@ -223,4 +215,3 @@ prevent_search(RefreshToken._meta.get_field('token'))
 prevent_search(OAuth2Application._meta.get_field('client_secret'))
 prevent_search(OAuth2Application._meta.get_field('client_id'))
 prevent_search(Grant._meta.get_field('code'))
-

awx/main/models/activity_stream.py

@@ -4,6 +4,7 @@
 # Tower
 from awx.api.versioning import reverse
 from awx.main.fields import JSONField
+from awx.main.models.base import accepts_json
 
 # Django
 from django.db import models
@@ -34,7 +35,7 @@ class ActivityStream(models.Model):
     actor = models.ForeignKey('auth.User', null=True, on_delete=models.SET_NULL, related_name='activity_stream')
     operation = models.CharField(max_length=13, choices=OPERATION_CHOICES)
     timestamp = models.DateTimeField(auto_now_add=True)
-    changes = models.TextField(blank=True)
+    changes = accepts_json(models.TextField(blank=True))
     deleted_actor = JSONField(null=True)
     action_node = models.CharField(
         blank=True,
@@ -66,6 +67,8 @@ class ActivityStream(models.Model):
     workflow_job_node = models.ManyToManyField("WorkflowJobNode", blank=True)
     workflow_job_template = models.ManyToManyField("WorkflowJobTemplate", blank=True)
     workflow_job = models.ManyToManyField("WorkflowJob", blank=True)
+    workflow_approval_template = models.ManyToManyField("WorkflowApprovalTemplate", blank=True)
+    workflow_approval = models.ManyToManyField("WorkflowApproval", blank=True)
     unified_job_template = models.ManyToManyField("UnifiedJobTemplate", blank=True, related_name='activity_stream_as_unified_job_template+')
     unified_job = models.ManyToManyField("UnifiedJob", blank=True, related_name='activity_stream_as_unified_job+')
     ad_hoc_command = models.ManyToManyField("AdHocCommand", blank=True)

awx/main/models/ad_hoc_commands.py

@@ -163,18 +163,18 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
             all_orgs.add(h.inventory.organization)
         active_templates = dict(error=set(),
                                 success=set(),
-                                any=set())
+                                started=set())
         base_notification_templates = NotificationTemplate.objects
         for org in all_orgs:
             for templ in base_notification_templates.filter(organization_notification_templates_for_errors=org):
                 active_templates['error'].add(templ)
             for templ in base_notification_templates.filter(organization_notification_templates_for_success=org):
                 active_templates['success'].add(templ)
-            for templ in base_notification_templates.filter(organization_notification_templates_for_any=org):
-                active_templates['any'].add(templ)
+            for templ in base_notification_templates.filter(organization_notification_templates_for_started=org):
+                active_templates['started'].add(templ)
         active_templates['error'] = list(active_templates['error'])
-        active_templates['any'] = list(active_templates['any'])
         active_templates['success'] = list(active_templates['success'])
+        active_templates['started'] = list(active_templates['started'])
         return active_templates
 
     def get_passwords_needed_to_start(self):

awx/main/models/base.py

@@ -386,14 +386,13 @@ class NotificationFieldsModel(BaseModel):
         related_name='%(class)s_notification_templates_for_success'
     )
 
-    notification_templates_any = models.ManyToManyField(
+    notification_templates_started = models.ManyToManyField(
         "NotificationTemplate",
         blank=True,
-        related_name='%(class)s_notification_templates_for_any'
+        related_name='%(class)s_notification_templates_for_started'
     )
 
 
-
 def prevent_search(relation):
     """
     Used to mark a model field or relation as "restricted from filtering"
@@ -409,3 +408,14 @@ def prevent_search(relation):
     """
     setattr(relation, '__prevent_search__', True)
     return relation
+
+
+def accepts_json(relation):
+    """
+    Used to mark a model field as allowing JSON e.g,. JobTemplate.extra_vars
+    This is *mostly* used as a way to provide type hints for certain fields
+    so that HTTP OPTIONS reports the type data we need for the CLI to allow
+    JSON/YAML input.
+    """
+    setattr(relation, '__accepts_json__', True)
+    return relation

awx/main/models/credential/__init__.py

@@ -105,10 +105,9 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     )
     inputs = CredentialInputField(
         blank=True,
-        default={},
-        help_text=_('Enter inputs using either JSON or YAML syntax. Use the '
-                    'radio button to toggle between the two. Refer to the '
-                    'Ansible Tower documentation for example syntax.')
+        default=dict,
+        help_text=_('Enter inputs using either JSON or YAML syntax. '
+                    'Refer to the Ansible Tower documentation for example syntax.')
     )
     admin_role = ImplicitRoleField(
         parent_role=[
@@ -343,17 +342,15 @@ class CredentialType(CommonModelNameNotUnique):
     )
     inputs = CredentialTypeInputField(
         blank=True,
-        default={},
-        help_text=_('Enter inputs using either JSON or YAML syntax. Use the '
-                    'radio button to toggle between the two. Refer to the '
-                    'Ansible Tower documentation for example syntax.')
+        default=dict,
+        help_text=_('Enter inputs using either JSON or YAML syntax. '
+                    'Refer to the Ansible Tower documentation for example syntax.')
     )
     injectors = CredentialTypeInjectorField(
         blank=True,
-        default={},
-        help_text=_('Enter injectors using either JSON or YAML syntax. Use the '
-                    'radio button to toggle between the two. Refer to the '
-                    'Ansible Tower documentation for example syntax.')
+        default=dict,
+        help_text=_('Enter injectors using either JSON or YAML syntax. '
+                    'Refer to the Ansible Tower documentation for example syntax.')
     )
 
     @classmethod
@@ -1117,7 +1114,7 @@ class CredentialInputSource(PrimordialModel):
     )
     metadata = DynamicCredentialInputField(
         blank=True,
-        default={}
+        default=dict
     )
 
     def clean_target_credential(self):

awx/main/models/credential/injectors.py

@@ -28,10 +28,7 @@ def gce(cred, env, private_data_dir):
     if 'INVENTORY_UPDATE_ID' not in env:
         env['GCE_EMAIL'] = username
         env['GCE_PROJECT'] = project
-    else:
-        # gcp_compute inventory plugin requires token_uri
-        # although it probably should not, since gce_modules do not
-        json_cred['token_uri'] = 'https://oauth2.googleapis.com/token'
+    json_cred['token_uri'] = 'https://oauth2.googleapis.com/token'
 
     handle, path = tempfile.mkstemp(dir=private_data_dir)
     f = os.fdopen(handle, 'w')
@@ -39,6 +36,14 @@ def gce(cred, env, private_data_dir):
     f.close()
     os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
     env['GCE_CREDENTIALS_FILE_PATH'] = path
+    env['GCP_SERVICE_ACCOUNT_FILE'] = path
+
+    # Handle env variables for new module types.
+    # This includes gcp_compute inventory plugin and
+    # all new gcp_* modules.
+    env['GCP_AUTH_KIND'] = 'serviceaccount'
+    env['GCP_PROJECT'] = project
+    env['GCP_ENV_TYPE'] = 'tower'
     return path
 
 

awx/main/models/events.py

@@ -149,7 +149,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
     )
     event_data = JSONField(
         blank=True,
-        default={},
+        default=dict,
     )
     failed = models.BooleanField(
         default=False,
@@ -567,7 +567,7 @@ class BaseCommandEvent(CreatedModifiedModel):
 
     event_data = JSONField(
         blank=True,
-        default={},
+        default=dict,
     )
     uuid = models.CharField(
         max_length=1024,
@@ -614,7 +614,13 @@ class BaseCommandEvent(CreatedModifiedModel):
             kwargs.pop('created', None)
 
         sanitize_event_keys(kwargs, cls.VALID_KEYS)
-        return cls.objects.create(**kwargs)
+        event = cls.objects.create(**kwargs)
+        if isinstance(event, AdHocCommandEvent):
+            analytics_logger.info(
+                'Event data saved.',
+                extra=dict(python_objects=dict(job_event=event))
+            )
+        return event
 
     def get_event_display(self):
         '''
@@ -622,6 +628,9 @@ class BaseCommandEvent(CreatedModifiedModel):
         '''
         return self.event
 
+    def get_event_display2(self):
+        return self.get_event_display()
+
     def get_host_status_counts(self):
         return create_host_status_counts(getattr(self, 'event_data', {}))
 

awx/main/models/inventory.py

@@ -45,7 +45,7 @@ from awx.main.models.base import (
     CommonModelNameNotUnique,
     VarsDictProperty,
     CLOUD_INVENTORY_SOURCES,
-    prevent_search
+    prevent_search, accepts_json
 )
 from awx.main.models.events import InventoryUpdateEvent
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
@@ -93,11 +93,11 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
         on_delete=models.SET_NULL,
         null=True,
     )
-    variables = models.TextField(
+    variables = accepts_json(models.TextField(
         blank=True,
         default='',
         help_text=_('Inventory variables in JSON or YAML format.'),
-    )
+    ))
     has_active_failures = models.BooleanField(
         default=False,
         editable=False,
@@ -309,7 +309,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
 
             # Now use in-memory maps to build up group info.
             all_group_names = []
-            for group in self.groups.only('name', 'id', 'variables'):
+            for group in self.groups.only('name', 'id', 'variables', 'inventory_id'):
                 group_info = dict()
                 if group.id in group_hosts_map:
                     group_info['hosts'] = group_hosts_map[group.id]
@@ -608,11 +608,11 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
         default='',
         help_text=_('The value used by the remote inventory source to uniquely identify the host'),
     )
-    variables = models.TextField(
+    variables = accepts_json(models.TextField(
         blank=True,
         default='',
         help_text=_('Host variables in JSON or YAML format.'),
-    )
+    ))
     last_job = models.ForeignKey(
         'Job',
         related_name='hosts_as_last_job+',
@@ -650,7 +650,7 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
     )
     ansible_facts = JSONBField(
         blank=True,
-        default={},
+        default=dict,
         help_text=_('Arbitrary JSON structure of most recent ansible_facts, per-host.'),
     )
     ansible_facts_modified = models.DateTimeField(
@@ -796,11 +796,11 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
         related_name='children',
         blank=True,
     )
-    variables = models.TextField(
+    variables = accepts_json(models.TextField(
         blank=True,
         default='',
         help_text=_('Group variables in JSON or YAML format.'),
-    )
+    ))
     hosts = models.ManyToManyField(
         'Host',
         related_name='groups',
@@ -1619,20 +1619,20 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
         base_notification_templates = NotificationTemplate.objects
         error_notification_templates = list(base_notification_templates
                                             .filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
+        started_notification_templates = list(base_notification_templates
+                                              .filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
         success_notification_templates = list(base_notification_templates
                                               .filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
-        any_notification_templates = list(base_notification_templates
-                                          .filter(unifiedjobtemplate_notification_templates_for_any__in=[self]))
         if self.inventory.organization is not None:
             error_notification_templates = set(error_notification_templates + list(base_notification_templates
                                                .filter(organization_notification_templates_for_errors=self.inventory.organization)))
+            started_notification_templates = set(started_notification_templates + list(base_notification_templates
+                                                 .filter(organization_notification_templates_for_started=self.inventory.organization)))
             success_notification_templates = set(success_notification_templates + list(base_notification_templates
                                                  .filter(organization_notification_templates_for_success=self.inventory.organization)))
-            any_notification_templates = set(any_notification_templates + list(base_notification_templates
-                                             .filter(organization_notification_templates_for_any=self.inventory.organization)))
         return dict(error=list(error_notification_templates),
-                    success=list(success_notification_templates),
-                    any=list(any_notification_templates))
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     def clean_source(self):  # TODO: remove in 3.3
         source = self.source
@@ -1991,6 +1991,8 @@ class azure_rm(PluginFileInjector):
 
         source_vars = inventory_update.source_vars_dict
 
+        ret['fail_on_template_errors'] = False
+
         group_by_hostvar = {
             'location': {'prefix': '', 'separator': '', 'key': 'location'},
             'tag': {'prefix': '', 'separator': '', 'key': 'tags.keys() | list if tags else []'},
@@ -2046,8 +2048,10 @@ class azure_rm(PluginFileInjector):
             'provisioning_state': 'provisioning_state | title',
             'computer_name': 'name',
             'type': 'resource_type',
-            'private_ip': 'private_ipv4_addresses[0]',
-            'public_ip': 'public_ipv4_addresses[0]',
+            'private_ip': 'private_ipv4_addresses[0] if private_ipv4_addresses else None',
+            'public_ip': 'public_ipv4_addresses[0] if public_ipv4_addresses else None',
+            'public_ip_name': 'public_ip_name if public_ip_name is defined else None',
+            'public_ip_id': 'public_ip_id if public_ip_id is defined else None',
             'tags': 'tags if tags else None'
         }
         # Special functionality from script
@@ -2330,6 +2334,12 @@ class gce(PluginFileInjector):
     ini_env_reference = 'GCE_INI_PATH'
     base_injector = 'managed'
 
+    def get_plugin_env(self, *args, **kwargs):
+        ret = super(gce, self).get_plugin_env(*args, **kwargs)
+        # We need native jinja2 types so that ip addresses can give JSON null value
+        ret['ANSIBLE_JINJA2_NATIVE'] = str(True)
+        return ret
+
     def get_script_env(self, inventory_update, private_data_dir, private_data_files):
         env = super(gce, self).get_script_env(inventory_update, private_data_dir, private_data_files)
         cred = inventory_update.get_cloud_credential()
@@ -2350,7 +2360,7 @@ class gce(PluginFileInjector):
             'gce_name': 'name',
             'gce_network': 'networkInterfaces[0].network.name',
             'gce_private_ip': 'networkInterfaces[0].networkIP',
-            'gce_public_ip': 'networkInterfaces[0].accessConfigs[0].natIP',
+            'gce_public_ip': 'networkInterfaces[0].accessConfigs[0].natIP | default(None)',
             'gce_status': 'status',
             'gce_subnetwork': 'networkInterfaces[0].subnetwork.name',
             'gce_tags': 'tags.get("items", [])',
@@ -2360,7 +2370,7 @@ class gce(PluginFileInjector):
             'gce_image': 'image',
             # We need this as long as hostnames is non-default, otherwise hosts
             # will not be addressed correctly, was returned in script
-            'ansible_ssh_host': 'networkInterfaces[0].accessConfigs[0].natIP'
+            'ansible_ssh_host': 'networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP)'
         }
 
     def inventory_as_dict(self, inventory_update, private_data_dir):

awx/main/models/jobs.py

@@ -27,7 +27,7 @@ from rest_framework.exceptions import ParseError
 from awx.api.versioning import reverse
 from awx.main.models.base import (
     BaseModel, CreatedModifiedModel,
-    prevent_search,
+    prevent_search, accepts_json,
     JOB_TYPE_CHOICES, VERBOSITY_CHOICES,
     VarsDictProperty
 )
@@ -96,6 +96,13 @@ class JobOptions(BaseModel):
         default='',
         blank=True,
     )
+    scm_branch = models.CharField(
+        max_length=1024,
+        default='',
+        blank=True,
+        help_text=_('Branch to use in job run. Project default used if blank. '
+                    'Only allowed if project allow_override field is set to true.'),
+    )
     forks = models.PositiveIntegerField(
         blank=True,
         default=0,
@@ -109,10 +116,10 @@ class JobOptions(BaseModel):
         blank=True,
         default=0,
     )
-    extra_vars = prevent_search(models.TextField(
+    extra_vars = prevent_search(accepts_json(models.TextField(
         blank=True,
         default='',
-    ))
+    )))
     job_tags = models.CharField(
         max_length=1024,
         blank=True,
@@ -234,6 +241,11 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         default=False,
         allows_field='credentials'
     )
+    ask_scm_branch_on_launch = AskForField(
+        blank=True,
+        default=False,
+        allows_field='scm_branch'
+    )
     job_slice_count = models.PositiveIntegerField(
         blank=True,
         default=1,
@@ -387,7 +399,21 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
                 # no-op case: Fields the same as template's value
                 # counted as neither accepted or ignored
                 continue
+            elif field_name == 'scm_branch' and old_value == '' and self.project and new_value == self.project.scm_branch:
+                # special case of "not provided" for branches
+                # job template does not provide branch, runs with default branch
+                continue
             elif getattr(self, ask_field_name):
+                # Special case where prompts can be rejected based on project setting
+                if field_name == 'scm_branch':
+                    if not self.project:
+                        rejected_data[field_name] = new_value
+                        errors_dict[field_name] = _('Project is missing.')
+                        continue
+                    if kwargs['scm_branch'] != self.project.scm_branch and not self.project.allow_override:
+                        rejected_data[field_name] = new_value
+                        errors_dict[field_name] = _('Project does not allow override of branch.')
+                        continue
                 # accepted prompt
                 prompted_data[field_name] = new_value
             else:
@@ -396,7 +422,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
                 # Not considered an error for manual launch, to support old
                 # behavior of putting them in ignored_fields and launching anyway
                 if 'prompts' not in exclude_errors:
-                    errors_dict[field_name] = _('Field is not configured to prompt on launch.').format(field_name=field_name)
+                    errors_dict[field_name] = _('Field is not configured to prompt on launch.')
 
         if ('prompts' not in exclude_errors and
                 (not getattr(self, 'ask_credential_on_launch', False)) and
@@ -435,19 +461,21 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         base_notification_templates = NotificationTemplate.objects
         error_notification_templates = list(base_notification_templates.filter(
             unifiedjobtemplate_notification_templates_for_errors__in=[self, self.project]))
+        started_notification_templates = list(base_notification_templates.filter(
+            unifiedjobtemplate_notification_templates_for_started__in=[self, self.project]))
         success_notification_templates = list(base_notification_templates.filter(
             unifiedjobtemplate_notification_templates_for_success__in=[self, self.project]))
-        any_notification_templates = list(base_notification_templates.filter(
-            unifiedjobtemplate_notification_templates_for_any__in=[self, self.project]))
         # Get Organization NotificationTemplates
         if self.project is not None and self.project.organization is not None:
             error_notification_templates = set(error_notification_templates + list(base_notification_templates.filter(
                 organization_notification_templates_for_errors=self.project.organization)))
+            started_notification_templates = set(started_notification_templates + list(base_notification_templates.filter(
+                organization_notification_templates_for_started=self.project.organization)))
             success_notification_templates = set(success_notification_templates + list(base_notification_templates.filter(
                 organization_notification_templates_for_success=self.project.organization)))
-            any_notification_templates = set(any_notification_templates + list(base_notification_templates.filter(
-                organization_notification_templates_for_any=self.project.organization)))
-        return dict(error=list(error_notification_templates), success=list(success_notification_templates), any=list(any_notification_templates))
+        return dict(error=list(error_notification_templates),
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     '''
     RelatedJobsMixin
@@ -483,7 +511,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
     )
     artifacts = JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     )
     scm_revision = models.CharField(
@@ -642,7 +670,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
         data = super(Job, self).notification_data()
         all_hosts = {}
         # NOTE: Probably related to job event slowness, remove at some point -matburt
-        if block:
+        if block and self.status != 'running':
             summaries = self.job_host_summaries.all()
             while block > 0 and not len(summaries):
                 time.sleep(1)
@@ -656,7 +684,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
                                           failures=h.failures,
                                           ok=h.ok,
                                           processed=h.processed,
-                                          skipped=h.skipped)
+                                          skipped=h.skipped)  # TODO: update with rescued, ignored (see https://github.com/ansible/awx/issues/4394)
         data.update(dict(inventory=self.inventory.name if self.inventory else None,
                          project=self.project.name if self.project else None,
                          playbook=self.playbook,
@@ -845,7 +873,7 @@ class LaunchTimeConfigBase(BaseModel):
     # This is a solution to the nullable CharField problem, specific to prompting
     char_prompts = JSONField(
         blank=True,
-        default={}
+        default=dict
     )
 
     def prompts_dict(self, display=False):
@@ -892,27 +920,6 @@ class LaunchTimeConfigBase(BaseModel):
     def display_extra_data(self):
         return self.display_extra_vars()
 
-    @property
-    def _credential(self):
-        '''
-        Only used for workflow nodes to support backward compatibility.
-        '''
-        try:
-            return [cred for cred in self.credentials.all() if cred.credential_type.kind == 'ssh'][0]
-        except IndexError:
-            return None
-
-    @property
-    def credential(self):
-        '''
-        Returns an integer so it can be used as IntegerField in serializer
-        '''
-        cred = self._credential
-        if cred is not None:
-            return cred.pk
-        else:
-            return None
-
 
 class LaunchTimeConfig(LaunchTimeConfigBase):
     '''
@@ -925,11 +932,11 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
     # Special case prompting fields, even more special than the other ones
     extra_data = JSONField(
         blank=True,
-        default={}
+        default=dict
     )
     survey_passwords = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     ))
     # Credentials needed for non-unified job / unified JT models
@@ -1133,13 +1140,13 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):
         base_notification_templates = NotificationTemplate.objects.all()
         error_notification_templates = list(base_notification_templates
                                             .filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
+        started_notification_templates = list(base_notification_templates
+                                              .filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
         success_notification_templates = list(base_notification_templates
                                               .filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
-        any_notification_templates = list(base_notification_templates
-                                          .filter(unifiedjobtemplate_notification_templates_for_any__in=[self]))
         return dict(error=list(error_notification_templates),
-                    success=list(success_notification_templates),
-                    any=list(any_notification_templates))
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     def _accept_or_ignore_job_kwargs(self, _exclude_errors=None, **kwargs):
         extra_data = kwargs.pop('extra_vars', {})

awx/main/models/mixins.py

@@ -100,7 +100,7 @@ class SurveyJobTemplateMixin(models.Model):
     )
     survey_spec = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
     ))
     ask_variables_on_launch = AskForField(
         blank=True,
@@ -360,7 +360,7 @@ class SurveyJobMixin(models.Model):
 
     survey_passwords = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     ))
 
@@ -483,4 +483,3 @@ class RelatedJobsMixin(object):
             raise RuntimeError("Programmer error. Expected _get_active_jobs() to return a QuerySet.")
 
         return [dict(id=t[0], type=mapping[t[1]]) for t in jobs.values_list('id', 'polymorphic_ctype_id')]
-

awx/main/models/notifications.py

@@ -2,13 +2,18 @@
 # All Rights Reserved.
 
 from copy import deepcopy
+import datetime
 import logging
+import json
 
 from django.db import models
 from django.conf import settings
 from django.core.mail.message import EmailMessage
+from django.db import connection
 from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import smart_str, force_text
+from jinja2 import sandbox
+from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
 
 # AWX
 from awx.api.versioning import reverse
@@ -44,7 +49,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
                           ('mattermost', _('Mattermost'), MattermostBackend),
                           ('rocketchat', _('Rocket.Chat'), RocketChatBackend),
                           ('irc', _('IRC'), IrcBackend)]
-    NOTIFICATION_TYPE_CHOICES = [(x[0], x[1]) for x in NOTIFICATION_TYPES]
+    NOTIFICATION_TYPE_CHOICES = sorted([(x[0], x[1]) for x in NOTIFICATION_TYPES])
     CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES])
 
     class Meta:
@@ -67,6 +72,45 @@ class NotificationTemplate(CommonModelNameNotUnique):
 
     notification_configuration = JSONField(blank=False)
 
+    def default_messages():
+        return {'started': None, 'success': None, 'error': None}
+
+    messages = JSONField(
+        null=True,
+        blank=True,
+        default=default_messages,
+        help_text=_('Optional custom messages for notification template.'))
+
+    def has_message(self, condition):
+        potential_template = self.messages.get(condition, {})
+        if potential_template == {}:
+            return False
+        if potential_template.get('message', {}) == {}:
+            return False
+        return True
+
+    def get_message(self, condition):
+        return self.messages.get(condition, {})
+
+    def build_notification_message(self, event_type, context):
+        env = sandbox.ImmutableSandboxedEnvironment()
+        templates = self.get_message(event_type)
+        msg_template = templates.get('message', {})
+
+        try:
+            notification_subject = env.from_string(msg_template).render(**context)
+        except (TemplateSyntaxError, UndefinedError, SecurityError):
+            notification_subject = ''
+
+
+        msg_body = templates.get('body', {})
+        try:
+            notification_body = env.from_string(msg_body).render(**context)
+        except (TemplateSyntaxError, UndefinedError, SecurityError):
+            notification_body = ''
+
+        return (notification_subject, notification_body)
+
     def get_absolute_url(self, request=None):
         return reverse('api:notification_template_detail', kwargs={'pk': self.pk}, request=request)
 
@@ -77,6 +121,26 @@ class NotificationTemplate(CommonModelNameNotUnique):
     def save(self, *args, **kwargs):
         new_instance = not bool(self.pk)
         update_fields = kwargs.get('update_fields', [])
+
+        # preserve existing notification messages if not overwritten by new messages
+        if not new_instance:
+            old_nt = NotificationTemplate.objects.get(pk=self.id)
+            old_messages = old_nt.messages
+            new_messages = self.messages
+
+            if old_messages is not None and new_messages is not None:
+                for event in ['started', 'success', 'error']:
+                    if not new_messages.get(event, {}) and old_messages.get(event, {}):
+                        new_messages[event] = old_messages[event]
+                        continue
+                    if new_messages.get(event, {}) and old_messages.get(event, {}):
+                        old_event_msgs = old_messages[event]
+                        new_event_msgs = new_messages[event]
+                        for msg_type in ['message', 'body']:
+                            if msg_type not in new_event_msgs and old_event_msgs.get(msg_type, None):
+                                new_event_msgs[msg_type] = old_event_msgs[msg_type]
+                    new_messages.setdefault(event, None)
+
         for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password",
                             self.notification_class.init_parameters):
             if self.notification_configuration[field].startswith("$encrypted$"):
@@ -117,9 +181,10 @@ class NotificationTemplate(CommonModelNameNotUnique):
     def send(self, subject, body):
         for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password",
                             self.notification_class.init_parameters):
-            self.notification_configuration[field] = decrypt_field(self,
-                                                                   'notification_configuration',
-                                                                   subfield=field)
+            if field in self.notification_configuration:
+                self.notification_configuration[field] = decrypt_field(self,
+                                                                       'notification_configuration',
+                                                                       subfield=field)
         recipients = self.notification_configuration.pop(self.notification_class.recipient_parameter)
         if not isinstance(recipients, list):
             recipients = [recipients]
@@ -129,7 +194,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
             if field not in notification_configuration:
                 if 'default' in params:
                     notification_configuration[field] = params['default']
-        backend_obj = self.notification_class(**notification_configuration) 
+        backend_obj = self.notification_class(**notification_configuration)
         notification_obj = EmailMessage(subject, backend_obj.format_body(body), sender, recipients)
         with set_environ(**settings.AWX_TASK_ENV):
             return backend_obj.send_messages([notification_obj])
@@ -199,48 +264,227 @@ class Notification(CreatedModifiedModel):
 
 
 class JobNotificationMixin(object):
+    STATUS_TO_TEMPLATE_TYPE = {'succeeded': 'success',
+                               'running': 'started',
+                               'failed': 'error'}
+    # Tree of fields that can be safely referenced in a notification message
+    JOB_FIELDS_WHITELIST = ['id', 'type', 'url', 'created', 'modified', 'name', 'description', 'job_type', 'playbook',
+                            'forks', 'limit', 'verbosity', 'job_tags', 'force_handlers', 'skip_tags', 'start_at_task',
+                            'timeout', 'use_fact_cache', 'launch_type', 'status', 'failed', 'started', 'finished',
+                            'elapsed', 'job_explanation', 'execution_node', 'controller_node', 'allow_simultaneous',
+                            'scm_revision', 'diff_mode', 'job_slice_number', 'job_slice_count', 'custom_virtualenv',
+                            {'host_status_counts': ['skipped', 'ok', 'changed', 'failures', 'dark']},
+                            {'playbook_counts': ['play_count', 'task_count']},
+                            {'summary_fields': [{'inventory': ['id', 'name', 'description', 'has_active_failures',
+                                                               'total_hosts', 'hosts_with_active_failures', 'total_groups',
+                                                               'groups_with_active_failures', 'has_inventory_sources',
+                                                               'total_inventory_sources', 'inventory_sources_with_failures',
+                                                               'organization_id', 'kind']},
+                                                {'project': ['id', 'name', 'description', 'status', 'scm_type']},
+                                                {'project_update': ['id', 'name', 'description', 'status', 'failed']},
+                                                {'job_template': ['id', 'name', 'description']},
+                                                {'unified_job_template': ['id', 'name', 'description', 'unified_job_type']},
+                                                {'instance_group': ['name', 'id']},
+                                                {'created_by': ['id', 'username', 'first_name', 'last_name']},
+                                                {'labels': ['count', 'results']},
+                                                {'source_workflow_job': ['description', 'elapsed', 'failed', 'id', 'name', 'status']}]}]
+
+    @classmethod
+    def context_stub(cls):
+        """Returns a stub context that can be used for validating notification messages.
+        Context has the same structure as the context that will actually be used to render
+        a notification message."""
+        context = {'job': {'allow_simultaneous': False,
+                           'controller_node': 'foo_controller',
+                           'created': datetime.datetime(2018, 11, 13, 6, 4, 0, 0, tzinfo=datetime.timezone.utc),
+                           'custom_virtualenv': 'my_venv',
+                           'description': 'Sample job description',
+                           'diff_mode': False,
+                           'elapsed': 0.403018,
+                           'execution_node': 'awx',
+                           'failed': False,
+                           'finished': False,
+                           'force_handlers': False,
+                           'forks': 0,
+                           'host_status_counts': {'skipped': 1, 'ok': 5, 'changed': 3, 'failures': 0, 'dark': 0},
+                           'id': 42,
+                           'job_explanation': 'Sample job explanation',
+                           'job_slice_count': 1,
+                           'job_slice_number': 0,
+                           'job_tags': '',
+                           'job_type': 'run',
+                           'launch_type': 'workflow',
+                           'limit': 'bar_limit',
+                           'modified': datetime.datetime(2018, 12, 13, 6, 4, 0, 0, tzinfo=datetime.timezone.utc),
+                           'name': 'Stub JobTemplate',
+                           'playbook_counts': {'play_count': 5, 'task_count': 10},
+                           'playbook': 'ping.yml',
+                           'scm_revision': '',
+                           'skip_tags': '',
+                           'start_at_task': '',
+                           'started': '2019-07-29T17:38:14.137461Z',
+                           'status': 'running',
+                           'summary_fields': {'created_by': {'first_name': '',
+                                                             'id': 1,
+                                                             'last_name': '',
+                                                             'username': 'admin'},
+                                              'instance_group': {'id': 1, 'name': 'tower'},
+                                              'inventory': {'description': 'Sample inventory description',
+                                                            'groups_with_active_failures': 0,
+                                                            'has_active_failures': False,
+                                                            'has_inventory_sources': False,
+                                                            'hosts_with_active_failures': 0,
+                                                            'id': 17,
+                                                            'inventory_sources_with_failures': 0,
+                                                            'kind': '',
+                                                            'name': 'Stub Inventory',
+                                                            'organization_id': 121,
+                                                            'total_groups': 0,
+                                                            'total_hosts': 1,
+                                                            'total_inventory_sources': 0},
+                                              'job_template': {'description': 'Sample job template description',
+                                                               'id': 39,
+                                                               'name': 'Stub JobTemplate'},
+                                              'labels': {'count': 0, 'results': []},
+                                              'project': {'description': 'Sample project description',
+                                                          'id': 38,
+                                                          'name': 'Stub project',
+                                                          'scm_type': 'git',
+                                                          'status': 'successful'},
+                                              'project_update': {'id': 5, 'name': 'Stub Project Update', 'description': 'Project Update',
+                                                                 'status': 'running', 'failed': False},
+                                              'unified_job_template': {'description': 'Sample unified job template description',
+                                                                       'id': 39,
+                                                                       'name': 'Stub Job Template',
+                                                                       'unified_job_type': 'job'},
+                                              'source_workflow_job': {'description': 'Sample workflow job description',
+                                                                      'elapsed': 0.000,
+                                                                      'failed': False,
+                                                                      'id': 88,
+                                                                      'name': 'Stub WorkflowJobTemplate',
+                                                                      'status': 'running'}},
+                           'timeout': 0,
+                           'type': 'job',
+                           'url': '/api/v2/jobs/13/',
+                           'use_fact_cache': False,
+                           'verbosity': 0},
+                   'job_friendly_name': 'Job',
+                   'url': 'https://towerhost/#/jobs/playbook/1010',
+                   'job_summary_dict': """{'url': 'https://towerhost/$/jobs/playbook/13',
+ 'traceback': '',
+ 'status': 'running',
+ 'started': '2019-08-07T21:46:38.362630+00:00',
+ 'project': 'Stub project',
+ 'playbook': 'ping.yml',
+ 'name': 'Stub Job Template',
+ 'limit': '',
+ 'inventory': 'Stub Inventory',
+ 'id': 42,
+ 'hosts': {},
+ 'friendly_name': 'Job',
+ 'finished': False,
+ 'credential': 'Stub credential',
+ 'created_by': 'admin'}"""}
+
+        return context
+
+    def context(self, serialized_job):
+        """Returns a context that can be used for rendering notification messages.
+        Context contains whitelisted content retrieved from a serialized job object
+        (see JobNotificationMixin.JOB_FIELDS_WHITELIST), the job's friendly name,
+        and a url to the job run."""
+        context = {'job': {},
+                   'job_friendly_name': self.get_notification_friendly_name(),
+                   'url': self.get_ui_url(),
+                   'job_summary_dict': json.dumps(self.notification_data(), indent=4)}
+
+        def build_context(node, fields, whitelisted_fields):
+            for safe_field in whitelisted_fields:
+                if type(safe_field) is dict:
+                    field, whitelist_subnode = safe_field.copy().popitem()
+                    # ensure content present in job serialization
+                    if field not in fields:
+                        continue
+                    subnode = fields[field]
+                    node[field] = {}
+                    build_context(node[field], subnode, whitelist_subnode)
+                else:
+                    # ensure content present in job serialization
+                    if safe_field not in fields:
+                        continue
+                    node[safe_field] = fields[safe_field]
+        build_context(context['job'], serialized_job, self.JOB_FIELDS_WHITELIST)
+
+        return context
+
     def get_notification_templates(self):
         raise RuntimeError("Define me")
 
     def get_notification_friendly_name(self):
         raise RuntimeError("Define me")
 
-    def _build_notification_message(self, status_str):
+    def notification_data(self):
+        raise RuntimeError("Define me")
+
+    def build_notification_message(self, nt, status):
+        env = sandbox.ImmutableSandboxedEnvironment()
+
+        from awx.api.serializers import UnifiedJobSerializer
+        job_serialization = UnifiedJobSerializer(self).to_representation(self)
+        context = self.context(job_serialization)
+
+        msg_template = body_template = None
+
+        if nt.messages:
+            templates = nt.messages.get(self.STATUS_TO_TEMPLATE_TYPE[status], {}) or {}
+            msg_template = templates.get('message', {})
+            body_template = templates.get('body', {})
+
+        if msg_template:
+            try:
+                notification_subject = env.from_string(msg_template).render(**context)
+            except (TemplateSyntaxError, UndefinedError, SecurityError):
+                notification_subject = ''
+        else:
+            notification_subject = u"{} #{} '{}' {}: {}".format(self.get_notification_friendly_name(),
+                                                                self.id,
+                                                                self.name,
+                                                                status,
+                                                                self.get_ui_url())
         notification_body = self.notification_data()
-        notification_subject = u"{} #{} '{}' {}: {}".format(self.get_notification_friendly_name(),
-                                                            self.id,
-                                                            self.name,
-                                                            status_str,
-                                                            notification_body['url'])
         notification_body['friendly_name'] = self.get_notification_friendly_name()
+        if body_template:
+            try:
+                notification_body['body'] = env.from_string(body_template).render(**context)
+            except (TemplateSyntaxError, UndefinedError, SecurityError):
+                notification_body['body'] = ''
+
         return (notification_subject, notification_body)
 
-    def build_notification_succeeded_message(self):
-        return self._build_notification_message('succeeded')
-
-    def build_notification_failed_message(self):
-        return self._build_notification_message('failed')
-
-    def send_notification_templates(self, status_str):
+    def send_notification_templates(self, status):
         from awx.main.tasks import send_notifications  # avoid circular import
-        if status_str not in ['succeeded', 'failed']:
-            raise ValueError(_("status_str must be either succeeded or failed"))
+        if status not in ['running', 'succeeded', 'failed']:
+            raise ValueError(_("status must be either running, succeeded or failed"))
         try:
             notification_templates = self.get_notification_templates()
         except Exception:
             logger.warn("No notification template defined for emitting notification")
-            notification_templates = None
-        if notification_templates:
-            if status_str == 'succeeded':
-                notification_template_type = 'success'
-            else:
-                notification_template_type = 'error'
-            all_notification_templates = set(notification_templates.get(notification_template_type, []) + notification_templates.get('any', []))
-            if len(all_notification_templates):
-                try:
-                    (notification_subject, notification_body) = getattr(self, 'build_notification_%s_message' % status_str)()
-                except AttributeError:
-                    raise NotImplementedError("build_notification_%s_message() does not exist" % status_str)
-                send_notifications.delay([n.generate_notification(notification_subject, notification_body).id
-                                          for n in all_notification_templates],
-                                         job_id=self.id)
+            return
+
+        if not notification_templates:
+            return
+
+        for nt in set(notification_templates.get(self.STATUS_TO_TEMPLATE_TYPE[status], [])):
+            try:
+                (notification_subject, notification_body) = self.build_notification_message(nt, status)
+            except AttributeError:
+                raise NotImplementedError("build_notification_message() does not exist" % status)
+
+            # Use kwargs to force late-binding
+            # https://stackoverflow.com/a/3431699/10669572
+            def send_it(local_nt=nt, local_subject=notification_subject, local_body=notification_body):
+                def _func():
+                    send_notifications.delay([local_nt.generate_notification(local_subject, local_body).id],
+                                             job_id=self.id)
+                return _func
+            connection.on_commit(send_it())

awx/main/models/oauth.py

@@ -98,8 +98,7 @@ class OAuth2AccessToken(AbstractAccessToken):
         related_name="%(app_label)s_%(class)s",
         help_text=_('The user representing the token owner')
     )
-    description = models.CharField(
-        max_length=200,
+    description = models.TextField(
         default='',
         blank=True,
     )

awx/main/models/organization.py

@@ -87,7 +87,10 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi
                      'execute_role', 'project_admin_role',
                      'inventory_admin_role', 'workflow_admin_role',
                      'notification_admin_role', 'credential_admin_role',
-                     'job_template_admin_role',],
+                     'job_template_admin_role', 'approval_role',],
+    )
+    approval_role = ImplicitRoleField(
+        parent_role='admin_role',
     )
 
 

awx/main/models/projects.py

@@ -106,6 +106,13 @@ class ProjectOptions(models.Model):
         verbose_name=_('SCM Branch'),
         help_text=_('Specific branch, tag or commit to checkout.'),
     )
+    scm_refspec = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        verbose_name=_('SCM refspec'),
+        help_text=_('For git projects, an additional refspec to fetch.'),
+    )
     scm_clean = models.BooleanField(
         default=False,
         help_text=_('Discard any local changes before syncing the project.'),
@@ -241,7 +248,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
     FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials']
     FIELDS_TO_DISCARD_AT_COPY = ['local_path']
-    FIELDS_TRIGGER_UPDATE = frozenset(['scm_url', 'scm_branch', 'scm_type'])
+    FIELDS_TRIGGER_UPDATE = frozenset(['scm_url', 'scm_branch', 'scm_type', 'scm_refspec'])
 
     class Meta:
         app_label = 'main'
@@ -261,9 +268,14 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     scm_update_cache_timeout = models.PositiveIntegerField(
         default=0,
         blank=True,
-        help_text=_('The number of seconds after the last project update ran that a new'
+        help_text=_('The number of seconds after the last project update ran that a new '
                     'project update will be launched as a job dependency.'),
     )
+    allow_override = models.BooleanField(
+        default=False,
+        help_text=_('Allow changing the SCM branch or revision in a job template '
+                    'that uses this project.'),
+    )
 
     scm_revision = models.CharField(
         max_length=1024,
@@ -411,24 +423,24 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         base_notification_templates = NotificationTemplate.objects
         error_notification_templates = list(base_notification_templates
                                             .filter(unifiedjobtemplate_notification_templates_for_errors=self))
+        started_notification_templates = list(base_notification_templates
+                                              .filter(unifiedjobtemplate_notification_templates_for_started=self))
         success_notification_templates = list(base_notification_templates
                                               .filter(unifiedjobtemplate_notification_templates_for_success=self))
-        any_notification_templates = list(base_notification_templates
-                                          .filter(unifiedjobtemplate_notification_templates_for_any=self))
         # Get Organization NotificationTemplates
         if self.organization is not None:
             error_notification_templates = set(error_notification_templates +
                                                list(base_notification_templates
                                                     .filter(organization_notification_templates_for_errors=self.organization)))
+            started_notification_templates = set(started_notification_templates +
+                                                 list(base_notification_templates
+                                                      .filter(organization_notification_templates_for_started=self.organization)))
             success_notification_templates = set(success_notification_templates +
                                                  list(base_notification_templates
                                                       .filter(organization_notification_templates_for_success=self.organization)))
-            any_notification_templates = set(any_notification_templates +
-                                             list(base_notification_templates
-                                                  .filter(organization_notification_templates_for_any=self.organization)))
         return dict(error=list(error_notification_templates),
-                    success=list(success_notification_templates),
-                    any=list(any_notification_templates))
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     def get_absolute_url(self, request=None):
         return reverse('api:project_detail', kwargs={'pk': self.pk}, request=request)
@@ -471,6 +483,14 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
         choices=PROJECT_UPDATE_JOB_TYPE_CHOICES,
         default='check',
     )
+    scm_revision = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        editable=False,
+        verbose_name=_('SCM Revision'),
+        help_text=_('The SCM Revision discovered by this update for the given project and branch.'),
+    )
 
     def _get_parent_field_name(self):
         return 'project'
@@ -567,5 +587,3 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
         if not selected_groups:
             return self.global_instance_groups
         return selected_groups
-
-

awx/main/models/rbac.py

@@ -48,6 +48,7 @@ role_names = {
     'read_role': _('Read'),
     'update_role': _('Update'),
     'use_role': _('Use'),
+    'approval_role': _('Approve'),
 }
 
 role_descriptions = {
@@ -70,6 +71,7 @@ role_descriptions = {
     'read_role': _('May view settings for the %s'),
     'update_role': _('May update the %s'),
     'use_role': _('Can use the %s in a job template'),
+    'approval_role': _('Can approve or deny a workflow approval node'),
 }
 
 

awx/main/models/unified_jobs.py

@@ -65,8 +65,8 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio
     # status inherits from related jobs. Thus, status must be able to be set to any status that a job status is settable to.
     JOB_STATUS_CHOICES = [
         ('new', _('New')),                  # Job has been created, but not started.
-        ('pending', _('Pending')),          # Job has been queued, but is not yet running.
-        ('waiting', _('Waiting')),          # Job is waiting on an update/dependency.
+        ('pending', _('Pending')),          # Job is pending Task Manager processing (blocked by dependency req, capacity or a concurrent job)
+        ('waiting', _('Waiting')),          # Job has been assigned to run on a specific node (and is about to run).
         ('running', _('Running')),          # Job is currently running.
         ('successful', _('Successful')),    # Job completed successfully.
         ('failed', _('Failed')),            # Job completed, but with failures.
@@ -641,7 +641,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
     )
     job_env = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     ))
     job_explanation = models.TextField(
@@ -1031,7 +1031,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
                 fd.write = lambda s: _write(smart_text(s))
 
                 cursor.copy_expert(
-                    "copy (select stdout from {} where {}={} order by start_line) to stdout".format(
+                    "copy (select stdout from {} where {}={} and stdout != '' order by start_line) to stdout".format(
                         self._meta.db_table + 'event',
                         self.event_parent_key,
                         self.id
@@ -1173,7 +1173,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
 
     def websocket_emit_data(self):
         ''' Return extra data that should be included when submitting data to the browser over the websocket connection '''
-        websocket_data = dict()
+        websocket_data = dict(type=self.get_real_instance_class()._meta.verbose_name.replace(' ', '_'))
         if self.spawned_by_workflow:
             websocket_data.update(dict(workflow_job_id=self.workflow_job_id,
                                        workflow_node_id=self.workflow_node_id))
@@ -1397,6 +1397,13 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
                 r['{}_user_email'.format(name)] = created_by.email
                 r['{}_user_first_name'.format(name)] = created_by.first_name
                 r['{}_user_last_name'.format(name)] = created_by.last_name
+
+        inventory = getattr_dne(self, 'inventory')
+        if inventory:
+            for name in ('awx', 'tower'):
+                r['{}_inventory_id'.format(name)] = inventory.pk
+                r['{}_inventory_name'.format(name)] = inventory.name
+
         return r
 
     def get_queue_name(self):

awx/main/models/workflow.py

@@ -13,7 +13,8 @@ from django.core.exceptions import ObjectDoesNotExist
 
 # AWX
 from awx.api.versioning import reverse
-from awx.main.models import prevent_search, UnifiedJobTemplate, UnifiedJob
+from awx.main.models import (prevent_search, accepts_json, UnifiedJobTemplate,
+                             UnifiedJob)
 from awx.main.models.notifications import (
     NotificationTemplate,
     JobNotificationMixin
@@ -34,11 +35,14 @@ from awx.main.models.jobs import LaunchTimeConfigBase, LaunchTimeConfig, JobTemp
 from awx.main.models.credential import Credential
 from awx.main.redact import REPLACE_STR
 from awx.main.fields import JSONField
+from awx.main.utils import schedule_task_manager
+
 
 from copy import copy
 from urllib.parse import urljoin
 
-__all__ = ['WorkflowJobTemplate', 'WorkflowJob', 'WorkflowJobOptions', 'WorkflowJobNode', 'WorkflowJobTemplateNode',]
+__all__ = ['WorkflowJobTemplate', 'WorkflowJob', 'WorkflowJobOptions', 'WorkflowJobNode',
+           'WorkflowJobTemplateNode', 'WorkflowApprovalTemplate', 'WorkflowApproval']
 
 
 logger = logging.getLogger('awx.main.models.workflow')
@@ -70,7 +74,7 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig):
     unified_job_template = models.ForeignKey(
         'UnifiedJobTemplate',
         related_name='%(class)ss',
-        blank=False,
+        blank=True,
         null=True,
         default=None,
         on_delete=models.SET_NULL,
@@ -160,6 +164,13 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
             new_node.credentials.add(cred)
         return new_node
 
+    def create_approval_template(self, **kwargs):
+        approval_template = WorkflowApprovalTemplate(**kwargs)
+        approval_template.save()
+        self.unified_job_template = approval_template
+        self.save()
+        return approval_template
+
 
 class WorkflowJobNode(WorkflowNodeBase):
     job = models.OneToOneField(
@@ -180,7 +191,7 @@ class WorkflowJobNode(WorkflowNodeBase):
     )
     ancestor_artifacts = JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     )
     do_not_run = models.BooleanField(
@@ -291,10 +302,10 @@ class WorkflowJobOptions(BaseModel):
     class Meta:
         abstract = True
 
-    extra_vars = prevent_search(models.TextField(
+    extra_vars = accepts_json(prevent_search(models.TextField(
         blank=True,
         default='',
-    ))
+    )))
     allow_simultaneous = models.BooleanField(
         default=False
     )
@@ -384,7 +395,11 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
     ])
     read_role = ImplicitRoleField(parent_role=[
         'singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR,
-        'organization.auditor_role', 'execute_role', 'admin_role'
+        'organization.auditor_role', 'execute_role', 'admin_role',
+        'approval_role',
+    ])
+    approval_role = ImplicitRoleField(parent_role=[
+        'organization.approval_role', 'admin_role',
     ])
 
     @property
@@ -419,13 +434,13 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
         base_notification_templates = NotificationTemplate.objects.all()
         error_notification_templates = list(base_notification_templates
                                             .filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
+        started_notification_templates = list(base_notification_templates
+                                              .filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
         success_notification_templates = list(base_notification_templates
                                               .filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
-        any_notification_templates = list(base_notification_templates
-                                          .filter(unifiedjobtemplate_notification_templates_for_any__in=[self]))
         return dict(error=list(error_notification_templates),
-                    success=list(success_notification_templates),
-                    any=list(any_notification_templates))
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     def create_unified_job(self, **kwargs):
         workflow_job = super(WorkflowJobTemplate, self).create_unified_job(**kwargs)
@@ -600,3 +615,92 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
         # WorkflowJobs don't _actually_ run anything in the dispatcher, so
         # there's no point in asking the dispatcher if it knows about this task
         return self.status == 'running'
+
+
+class WorkflowApprovalTemplate(UnifiedJobTemplate):
+
+    FIELDS_TO_PRESERVE_AT_COPY = ['description', 'timeout',]
+
+    class Meta:
+        app_label = 'main'
+
+    timeout = models.IntegerField(
+        blank=True,
+        default=0,
+        help_text=_("The amount of time (in seconds) before the approval node expires and fails."),
+    )
+
+    @classmethod
+    def _get_unified_job_class(cls):
+        return WorkflowApproval
+
+    @classmethod
+    def _get_unified_job_field_names(cls):
+        return ['name', 'description', 'timeout']
+
+    def get_absolute_url(self, request=None):
+        return reverse('api:workflow_approval_template_detail', kwargs={'pk': self.pk}, request=request)
+
+    @property
+    def workflow_job_template(self):
+        return self.workflowjobtemplatenodes.first().workflow_job_template
+
+
+class WorkflowApproval(UnifiedJob):
+    class Meta:
+        app_label = 'main'
+
+    workflow_approval_template = models.ForeignKey(
+        'WorkflowApprovalTemplate',
+        related_name='approvals',
+        blank=True,
+        null=True,
+        default=None,
+        on_delete=models.SET_NULL,
+    )
+    timeout = models.IntegerField(
+        blank=True,
+        default=0,
+        help_text=_("The amount of time (in seconds) before the approval node expires and fails."),
+    )
+    timed_out = models.BooleanField(
+        default=False,
+        help_text=_("Shows when an approval node (with a timeout assigned to it) has timed out.")
+    )
+
+
+    @classmethod
+    def _get_unified_job_template_class(cls):
+        return WorkflowApprovalTemplate
+
+    def get_absolute_url(self, request=None):
+        return reverse('api:workflow_approval_detail', kwargs={'pk': self.pk}, request=request)
+
+    @property
+    def event_class(self):
+        return None
+
+    def _get_parent_field_name(self):
+        return 'workflow_approval_template'
+
+    def approve(self, request=None):
+        self.status = 'successful'
+        self.save()
+        self.websocket_emit_status(self.status)
+        schedule_task_manager()
+        return reverse('api:workflow_approval_approve', kwargs={'pk': self.pk}, request=request)
+
+    def deny(self, request=None):
+        self.status = 'failed'
+        self.save()
+        self.websocket_emit_status(self.status)
+        schedule_task_manager()
+        return reverse('api:workflow_approval_deny', kwargs={'pk': self.pk}, request=request)
+
+    @property
+    def workflow_job_template(self):
+        return self.unified_job_node.workflow_job.unified_job_template
+
+    @property
+    def workflow_job(self):
+        return self.unified_job_node.workflow_job

awx/main/notifications/email_backend.py

@@ -19,6 +19,12 @@ class CustomEmailBackend(EmailBackend):
                        "sender": {"label": "Sender Email", "type": "string"},
                        "recipients": {"label": "Recipient List", "type": "list"},
                        "timeout": {"label": "Timeout", "type": "int", "default": 30}}
+
+    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    DEFAULT_BODY = smart_text(_("{{ job_friendly_name }} #{{ job.id }} had status {{ job.status }}, view details at {{ url }}\n\n{{ job_summary_dict }}"))
+    default_messages = {"started": {"message": DEFAULT_SUBJECT, "body": DEFAULT_BODY},
+                        "success": {"message": DEFAULT_SUBJECT, "body": DEFAULT_BODY},
+                        "error": {"message": DEFAULT_SUBJECT, "body": DEFAULT_BODY}}
     recipient_parameter = "recipients"
     sender_parameter = "sender"
 

awx/main/notifications/grafana_backend.py

@@ -21,6 +21,11 @@ class GrafanaBackend(AWXBaseEmailBackend):
     recipient_parameter = "grafana_url"
     sender_parameter = None
 
+    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    default_messages = {"started": {"message": DEFAULT_SUBJECT},
+                        "success": {"message": DEFAULT_SUBJECT},
+                        "error": {"message": DEFAULT_SUBJECT}}
+
     def __init__(self, grafana_key,dashboardId=None, panelId=None, annotation_tags=None, grafana_no_verify_ssl=False, isRegion=True,
                  fail_silently=False, **kwargs):
         super(GrafanaBackend, self).__init__(fail_silently=fail_silently)

awx/main/notifications/hipchat_backend.py

@@ -23,6 +23,11 @@ class HipChatBackend(AWXBaseEmailBackend):
     recipient_parameter = "rooms"
     sender_parameter = "message_from"
 
+    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    default_messages = {"started": {"message": DEFAULT_SUBJECT},
+                        "success": {"message": DEFAULT_SUBJECT},
+                        "error": {"message": DEFAULT_SUBJECT}}
+
     def __init__(self, token, color, api_url, notify, fail_silently=False, **kwargs):
         super(HipChatBackend, self).__init__(fail_silently=fail_silently)
         self.token = token

awx/main/notifications/irc_backend.py

@@ -25,6 +25,11 @@ class IrcBackend(AWXBaseEmailBackend):
     recipient_parameter = "targets"
     sender_parameter = None
 
+    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    default_messages = {"started": {"message": DEFAULT_SUBJECT},
+                        "success": {"message": DEFAULT_SUBJECT},
+                        "error": {"message": DEFAULT_SUBJECT}}
+
     def __init__(self, server, port, nickname, password, use_ssl, fail_silently=False, **kwargs):
         super(IrcBackend, self).__init__(fail_silently=fail_silently)
         self.server = server

awx/main/notifications/mattermost_backend.py

@@ -19,6 +19,11 @@ class MattermostBackend(AWXBaseEmailBackend):
     recipient_parameter = "mattermost_url"
     sender_parameter = None
 
+    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    default_messages = {"started": {"message": DEFAULT_SUBJECT},
+                        "success": {"message": DEFAULT_SUBJECT},
+                        "error": {"message": DEFAULT_SUBJECT}}
+
     def __init__(self, mattermost_no_verify_ssl=False, mattermost_channel=None, mattermost_username=None,
                  mattermost_icon_url=None, fail_silently=False, **kwargs):
         super(MattermostBackend, self).__init__(fail_silently=fail_silently)

awx/main/notifications/pagerduty_backend.py

@@ -20,6 +20,12 @@ class PagerDutyBackend(AWXBaseEmailBackend):
     recipient_parameter = "service_key"
     sender_parameter = "client_name"
 
+    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    DEFAULT_BODY = "{{ job_summary_dict }}"
+    default_messages = {"started": { "message": DEFAULT_SUBJECT, "body": DEFAULT_BODY},
+                        "success": { "message": DEFAULT_SUBJECT, "body": DEFAULT_BODY},
+                        "error": { "message": DEFAULT_SUBJECT, "body": DEFAULT_BODY}}
+
     def __init__(self, subdomain, token, fail_silently=False, **kwargs):
         super(PagerDutyBackend, self).__init__(fail_silently=fail_silently)
         self.subdomain = subdomain

awx/main/notifications/rocketchat_backend.py

@@ -19,6 +19,11 @@ class RocketChatBackend(AWXBaseEmailBackend):
     recipient_parameter = "rocketchat_url"
     sender_parameter = None
 
+    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    default_messages = {"started": {"message": DEFAULT_SUBJECT},
+                        "success": {"message": DEFAULT_SUBJECT},
+                        "error": {"message": DEFAULT_SUBJECT}}
+
     def __init__(self, rocketchat_no_verify_ssl=False, rocketchat_username=None, rocketchat_icon_url=None, fail_silently=False, **kwargs):
         super(RocketChatBackend, self).__init__(fail_silently=fail_silently)
         self.rocketchat_no_verify_ssl = rocketchat_no_verify_ssl

awx/main/notifications/slack_backend.py

@@ -19,6 +19,11 @@ class SlackBackend(AWXBaseEmailBackend):
     recipient_parameter = "channels"
     sender_parameter = None
 
+    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    default_messages = {"started": {"message": DEFAULT_SUBJECT},
+                        "success": {"message": DEFAULT_SUBJECT},
+                        "error": {"message": DEFAULT_SUBJECT}}
+
     def __init__(self, token, hex_color="", fail_silently=False, **kwargs):
         super(SlackBackend, self).__init__(fail_silently=fail_silently)
         self.token = token
@@ -50,7 +55,7 @@ class SlackBackend(AWXBaseEmailBackend):
                     if ret['ok']:
                         sent_messages += 1
                     else:
-                        raise RuntimeError("Slack Notification unable to send {}: {}".format(r, m.subject))
+                        raise RuntimeError("Slack Notification unable to send {}: {} ({})".format(r, m.subject, ret['error']))
             except Exception as e:
                 logger.error(smart_text(_("Exception sending messages: {}").format(e)))
                 if not self.fail_silently:

awx/main/notifications/twilio_backend.py

@@ -21,6 +21,11 @@ class TwilioBackend(AWXBaseEmailBackend):
     recipient_parameter = "to_numbers"
     sender_parameter = "from_number"
 
+    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    default_messages = {"started": {"message": DEFAULT_SUBJECT},
+                        "success": {"message": DEFAULT_SUBJECT},
+                        "error": {"message": DEFAULT_SUBJECT}}
+
     def __init__(self, account_sid, account_token, fail_silently=False, **kwargs):
         super(TwilioBackend, self).__init__(fail_silently=fail_silently)
         self.account_sid = account_sid

awx/main/notifications/webhook_backend.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2016 Ansible, Inc.
 # All Rights Reserved.
 
+import json
 import logging
 import requests
 
@@ -15,25 +16,52 @@ logger = logging.getLogger('awx.main.notifications.webhook_backend')
 class WebhookBackend(AWXBaseEmailBackend):
 
     init_parameters = {"url": {"label": "Target URL", "type": "string"},
+                       "http_method": {"label": "HTTP Method", "type": "string", "default": "POST"},
                        "disable_ssl_verification": {"label": "Verify SSL", "type": "bool", "default": False},
+                       "username": {"label": "Username", "type": "string", "default": ""},
+                       "password": {"label": "Password", "type": "password", "default": ""},
                        "headers": {"label": "HTTP Headers", "type": "object"}}
     recipient_parameter = "url"
     sender_parameter = None
 
-    def __init__(self, headers, disable_ssl_verification=False, fail_silently=False, **kwargs):
+    DEFAULT_BODY = "{{ job_summary_dict }}"
+    default_messages = {"started": {"body": DEFAULT_BODY},
+                        "success": {"body": DEFAULT_BODY},
+                        "error": {"body": DEFAULT_BODY}}
+
+    def __init__(self, http_method, headers, disable_ssl_verification=False, fail_silently=False, username=None, password=None, **kwargs):
+        self.http_method = http_method
         self.disable_ssl_verification = disable_ssl_verification
         self.headers = headers
+        self.username = username
+        self.password = password
         super(WebhookBackend, self).__init__(fail_silently=fail_silently)
 
     def format_body(self, body):
+        # If `body` has body field, attempt to use this as the main body,
+        # otherwise, leave it as a sub-field
+        if isinstance(body, dict) and 'body' in body and isinstance(body['body'], str):
+            try:
+                potential_body = json.loads(body['body'])
+                if isinstance(potential_body, dict):
+                    body = potential_body
+            except json.JSONDecodeError:
+                pass
         return body
 
     def send_messages(self, messages):
         sent_messages = 0
         if 'User-Agent' not in self.headers:
             self.headers['User-Agent'] = "Tower {}".format(get_awx_version())
+        if self.http_method.lower() not in ['put','post']:
+            raise ValueError("HTTP method must be either 'POST' or 'PUT'.")
+        chosen_method = getattr(requests, self.http_method.lower(), None)
         for m in messages:
-            r = requests.post("{}".format(m.recipients()[0]),
+            auth = None
+            if self.username or self.password:
+                auth = (self.username, self.password)
+            r = chosen_method("{}".format(m.recipients()[0]),
+                              auth=auth,
                               json=m.body,
                               headers=self.headers,
                               verify=(not self.disable_ssl_verification))

awx/main/scheduler/task_manager.py

@@ -23,6 +23,7 @@ from awx.main.models import (
     Project,
     ProjectUpdate,
     SystemJob,
+    WorkflowApproval,
     WorkflowJob,
     WorkflowJobTemplate
 )
@@ -193,6 +194,8 @@ class TaskManager():
                 status_changed = True
             if status_changed:
                 workflow_job.websocket_emit_status(workflow_job.status)
+                # Operations whose queries rely on modifications made during the atomic scheduling session
+                workflow_job.send_notification_templates('succeeded' if workflow_job.status == 'successful' else 'failed')
                 if workflow_job.spawned_by_workflow:
                     schedule_task_manager()
         return result
@@ -233,6 +236,7 @@ class TaskManager():
         else:
             if type(task) is WorkflowJob:
                 task.status = 'running'
+                task.send_notification_templates('running')
                 logger.debug('Transitioning %s to running status.', task.log_format)
                 schedule_task_manager()
             elif not task.supports_isolation() and rampart_group.controller_id:
@@ -515,6 +519,24 @@ class TaskManager():
             if not found_acceptable_queue:
                 logger.debug("{} couldn't be scheduled on graph, waiting for next cycle".format(task.log_format))
 
+    def timeout_approval_node(self):
+        workflow_approvals = WorkflowApproval.objects.filter(status='pending')
+        now = tz_now()
+        for task in workflow_approvals:
+            approval_timeout_seconds = timedelta(seconds=task.timeout)
+            if task.timeout == 0:
+                continue
+            if (now - task.created) >= approval_timeout_seconds:
+                timeout_message = _(
+                    "The approval node {name} ({pk}) has expired after {timeout} seconds."
+                ).format(name=task.name, pk=task.pk, timeout=task.timeout)
+                logger.warn(timeout_message)
+                task.timed_out = True
+                task.status = 'failed'
+                task.websocket_emit_status(task.status)
+                task.job_explanation = timeout_message
+                task.save(update_fields=['status', 'job_explanation', 'timed_out'])
+
     def calculate_capacity_consumed(self, tasks):
         self.graph = InstanceGroup.objects.capacity_values(tasks=tasks, graph=self.graph)
 
@@ -570,6 +592,8 @@ class TaskManager():
 
             self.spawn_workflow_graph_jobs(running_workflow_tasks)
 
+            self.timeout_approval_node()
+
             self.process_tasks(all_sorted_tasks)
         return finished_wfjs
 
@@ -581,10 +605,5 @@ class TaskManager():
                     logger.debug("Not running scheduler, another task holds lock")
                     return
                 logger.debug("Starting Scheduler")
-
                 with task_manager_bulk_reschedule():
-                    finished_wfjs = self._schedule()
-
-                # Operations whose queries rely on modifications made during the atomic scheduling session
-                for wfj in WorkflowJob.objects.filter(id__in=finished_wfjs):
-                    wfj.send_notification_templates('succeeded' if wfj.status == 'successful' else 'failed')
+                    self._schedule()

awx/main/signals.py

@@ -34,8 +34,8 @@ from awx.main.models import (
     InventorySource, InventoryUpdateEvent, Job, JobEvent, JobHostSummary,
     JobTemplate, OAuth2AccessToken, Organization, Project, ProjectUpdateEvent,
     Role, SystemJob, SystemJobEvent, SystemJobTemplate, UnifiedJob,
-    UnifiedJobTemplate, User, UserSessionMembership,
-    ROLE_SINGLETON_SYSTEM_ADMINISTRATOR
+    UnifiedJobTemplate, User, UserSessionMembership, WorkflowJobTemplateNode,
+    WorkflowApproval, WorkflowApprovalTemplate, ROLE_SINGLETON_SYSTEM_ADMINISTRATOR
 )
 from awx.main.constants import CENSOR_VALUE
 from awx.main.utils import model_instance_diff, model_to_dict, camelcase_to_underscore, get_current_apps
@@ -355,6 +355,7 @@ def update_host_last_job_after_job_deleted(sender, **kwargs):
     for host in Host.objects.filter(pk__in=hosts_pks):
         _update_host_last_jhs(host)
 
+
 # Set via ActivityStreamRegistrar to record activity stream events
 
 
@@ -429,6 +430,8 @@ def model_serializer_mapping():
         models.Label: serializers.LabelSerializer,
         models.WorkflowJobTemplate: serializers.WorkflowJobTemplateWithSpecSerializer,
         models.WorkflowJobTemplateNode: serializers.WorkflowJobTemplateNodeSerializer,
+        models.WorkflowApproval: serializers.WorkflowApprovalActivityStreamSerializer,
+        models.WorkflowApprovalTemplate: serializers.WorkflowApprovalTemplateSerializer,
         models.WorkflowJob: serializers.WorkflowJobSerializer,
         models.OAuth2AccessToken: serializers.OAuth2TokenSerializer,
         models.OAuth2Application: serializers.OAuth2ApplicationSerializer,
@@ -637,6 +640,30 @@ def delete_inventory_for_org(sender, instance, **kwargs):
             logger.debug(e)
 
 
+@receiver(pre_delete, sender=WorkflowJobTemplateNode)
+def delete_approval_templates(sender, instance, **kwargs):
+    if type(instance.unified_job_template) is WorkflowApprovalTemplate:
+        instance.unified_job_template.delete()
+
+
+@receiver(pre_save, sender=WorkflowJobTemplateNode)
+def delete_approval_node_type_change(sender, instance, **kwargs):
+    try:
+        old = WorkflowJobTemplateNode.objects.get(id=instance.id)
+    except sender.DoesNotExist:
+        return
+    if old.unified_job_template == instance.unified_job_template:
+        return
+    if type(old.unified_job_template) is WorkflowApprovalTemplate:
+        old.unified_job_template.delete()
+
+
+@receiver(pre_delete, sender=WorkflowApprovalTemplate)
+def deny_orphaned_approvals(sender, instance, **kwargs):
+    for approval in WorkflowApproval.objects.filter(workflow_approval_template=instance, status='pending'):
+        approval.deny()
+
+
 @receiver(post_save, sender=Session)
 def save_user_session_membership(sender, **kwargs):
     session = kwargs.get('instance', None)

awx/main/tasks.py

@@ -20,6 +20,8 @@ from distutils.dir_util import copy_tree
 from distutils.version import LooseVersion as Version
 import yaml
 import fcntl
+from pathlib import Path
+from uuid import uuid4
 try:
     import psutil
 except Exception:
@@ -41,6 +43,10 @@ from django.core.exceptions import ObjectDoesNotExist
 # Django-CRUM
 from crum import impersonate
 
+# GitPython
+import git
+from gitdb.exc import BadName as BadGitName
+
 # Runner
 import ansible_runner
 
@@ -67,7 +73,7 @@ from awx.main.utils import (get_ssh_version, update_scm_url,
                             ignore_inventory_computed_fields,
                             ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager,
                             get_awx_version)
-from awx.main.utils.common import _get_ansible_version, get_custom_venv_choices
+from awx.main.utils.common import get_ansible_version, _get_ansible_version, get_custom_venv_choices
 from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja
 from awx.main.utils.reload import stop_local_services
 from awx.main.utils.pglock import advisory_lock
@@ -330,10 +336,12 @@ def send_notifications(notification_list, job_id=None):
 
 @task()
 def gather_analytics():
-    if settings.PENDO_TRACKING_STATE == 'off':
+    if not settings.INSIGHTS_TRACKING_STATE:
         return
     try:
         tgz = analytics.gather()
+        if not tgz:
+            return
         logger.debug('gathered analytics: {}'.format(tgz))
         analytics.ship(tgz)
     finally:
@@ -615,7 +623,7 @@ def update_smart_memberships_for_inventory(smart_inventory):
                     SmartInventoryMembership(inventory_id=smart_inventory.id, host_id=host_id)
                     for host_id in additions
                 ]
-                SmartInventoryMembership.objects.bulk_create(add_for_inventory)
+                SmartInventoryMembership.objects.bulk_create(add_for_inventory, ignore_conflicts=True)
         logger.debug('Smart host membership cached for {}, {} additions, {} removals, {} total count.'.format(
             smart_inventory.pk, len(additions), len(removals), len(new)
         ))
@@ -692,9 +700,11 @@ class BaseTask(object):
     model = None
     event_model = None
     abstract = True
-    cleanup_paths = []
     proot_show_paths = []
 
+    def __init__(self):
+        self.cleanup_paths = []
+
     def update_model(self, pk, _attempt=0, **updates):
         """Reload the model instance from the database and update the
         given fields.
@@ -767,9 +777,11 @@ class BaseTask(object):
         os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
         if settings.AWX_CLEANUP_PATHS:
             self.cleanup_paths.append(path)
-        # Ansible Runner requires that this directory exists.
-        # Specifically, when using process isolation
-        os.mkdir(os.path.join(path, 'project'))
+        runner_project_folder = os.path.join(path, 'project')
+        if not os.path.exists(runner_project_folder):
+            # Ansible Runner requires that this directory exists.
+            # Specifically, when using process isolation
+            os.mkdir(runner_project_folder)
         return path
 
     def build_private_data_files(self, instance, private_data_dir):
@@ -827,8 +839,10 @@ class BaseTask(object):
                     os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
                 private_data_files['credentials'][credential] = path
             for credential, data in private_data.get('certificates', {}).items():
-                name = 'credential_%d-cert.pub' % credential.pk
-                path = os.path.join(private_data_dir, name)
+                artifact_dir = os.path.join(private_data_dir, 'artifacts', str(self.instance.id))
+                if not os.path.exists(artifact_dir):
+                    os.makedirs(artifact_dir, mode=0o700)
+                path = os.path.join(artifact_dir, 'ssh_key_data-cert.pub')
                 with open(path, 'w') as f:
                     f.write(data)
                     f.close()
@@ -856,10 +870,29 @@ class BaseTask(object):
         '''
         process_isolation_params = dict()
         if self.should_use_proot(instance):
+            local_paths = [private_data_dir]
+            if cwd != private_data_dir and Path(private_data_dir) not in Path(cwd).parents:
+                local_paths.append(cwd)
+            show_paths = self.proot_show_paths + local_paths + \
+                settings.AWX_PROOT_SHOW_PATHS
+
+            # Help the user out by including the collections path inside the bubblewrap environment
+            if getattr(settings, 'AWX_ANSIBLE_COLLECTIONS_PATHS', []):
+                show_paths.extend(settings.AWX_ANSIBLE_COLLECTIONS_PATHS)
+
+            pi_path = settings.AWX_PROOT_BASE_PATH
+            if not self.instance.is_isolated():
+                pi_path = tempfile.mkdtemp(
+                    prefix='ansible_runner_pi_',
+                    dir=settings.AWX_PROOT_BASE_PATH
+                )
+                os.chmod(pi_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+                self.cleanup_paths.append(pi_path)
+
             process_isolation_params = {
                 'process_isolation': True,
-                'process_isolation_path': settings.AWX_PROOT_BASE_PATH,
-                'process_isolation_show_paths': self.proot_show_paths + [private_data_dir, cwd] + settings.AWX_PROOT_SHOW_PATHS,
+                'process_isolation_path': pi_path,
+                'process_isolation_show_paths': show_paths,
                 'process_isolation_hide_paths': [
                     settings.AWX_PROOT_BASE_PATH,
                     '/etc/tower',
@@ -933,6 +966,11 @@ class BaseTask(object):
         if self.should_use_proot(instance):
             env['PROOT_TMP_DIR'] = settings.AWX_PROOT_BASE_PATH
         env['AWX_PRIVATE_DATA_DIR'] = private_data_dir
+
+        if 'ANSIBLE_COLLECTIONS_PATHS' in env:
+            env['ANSIBLE_COLLECTIONS_PATHS'] += os.pathsep + os.pathsep.join(settings.AWX_ANSIBLE_COLLECTIONS_PATHS)
+        else:
+            env['ANSIBLE_COLLECTIONS_PATHS'] = os.pathsep.join(settings.AWX_ANSIBLE_COLLECTIONS_PATHS)
         return env
 
     def should_use_proot(self, instance):
@@ -1005,7 +1043,7 @@ class BaseTask(object):
             expect_passwords[k] = passwords.get(v, '') or ''
         return expect_passwords
 
-    def pre_run_hook(self, instance):
+    def pre_run_hook(self, instance, private_data_dir):
         '''
         Hook for any steps to run before the job/task starts
         '''
@@ -1131,7 +1169,9 @@ class BaseTask(object):
 
         try:
             isolated = self.instance.is_isolated()
-            self.pre_run_hook(self.instance)
+            self.instance.send_notification_templates("running")
+            private_data_dir = self.build_private_data_dir(self.instance)
+            self.pre_run_hook(self.instance, private_data_dir)
             if self.instance.cancel_flag:
                 self.instance = self.update_model(self.instance.pk, status='canceled')
             if self.instance.status != 'running':
@@ -1147,7 +1187,6 @@ class BaseTask(object):
             # store a record of the venv used at runtime
             if hasattr(self.instance, 'custom_virtualenv'):
                 self.update_model(pk, custom_virtualenv=getattr(self.instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH))
-            private_data_dir = self.build_private_data_dir(self.instance)
 
             # Fetch "cached" fact data from prior runs and put on the disk
             # where ansible expects to find it
@@ -1230,9 +1269,6 @@ class BaseTask(object):
                     module_args = ansible_runner.utils.args2cmdline(
                         params.get('module_args'),
                     )
-                else:
-                    # otherwise, it's a playbook, so copy the project dir
-                    copy_tree(cwd, os.path.join(private_data_dir, 'project'))
                 shutil.move(
                     params.pop('inventory'),
                     os.path.join(private_data_dir, 'inventory')
@@ -1438,6 +1474,15 @@ class RunJob(BaseTask):
             if authorize:
                 env['ANSIBLE_NET_AUTH_PASS'] = network_cred.get_input('authorize_password', default='')
 
+        for env_key, folder in (
+                ('ANSIBLE_COLLECTIONS_PATHS', 'requirements_collections'),
+                ('ANSIBLE_ROLES_PATH', 'requirements_roles')):
+            paths = []
+            if env_key in env:
+                paths.append(env[env_key])
+            paths.append(os.path.join(private_data_dir, folder))
+            env[env_key] = os.pathsep.join(paths)
+
         return env
 
     def build_args(self, job, private_data_dir, passwords):
@@ -1506,15 +1551,10 @@ class RunJob(BaseTask):
         return args
 
     def build_cwd(self, job, private_data_dir):
-        cwd = job.project.get_project_path()
-        if not cwd:
-            root = settings.PROJECTS_ROOT
-            raise RuntimeError('project local_path %s cannot be found in %s' %
-                               (job.project.local_path, root))
-        return cwd
+        return os.path.join(private_data_dir, 'project')
 
     def build_playbook_path_relative_to_cwd(self, job, private_data_dir):
-        return os.path.join(job.playbook)
+        return job.playbook
 
     def build_extra_vars_file(self, job, private_data_dir):
         # Define special extra_vars for AWX, combine with job.extra_vars.
@@ -1561,39 +1601,86 @@ class RunJob(BaseTask):
         '''
         return getattr(settings, 'AWX_PROOT_ENABLED', False)
 
-    def pre_run_hook(self, job):
+    def pre_run_hook(self, job, private_data_dir):
         if job.inventory is None:
             error = _('Job could not start because it does not have a valid inventory.')
             self.update_model(job.pk, status='failed', job_explanation=error)
             raise RuntimeError(error)
-        if job.project and job.project.scm_type:
+        elif job.project is None:
+            error = _('Job could not start because it does not have a valid project.')
+            self.update_model(job.pk, status='failed', job_explanation=error)
+            raise RuntimeError(error)
+        elif job.project.status in ('error', 'failed'):
+            msg = _(
+                'The project revision for this job template is unknown due to a failed update.'
+            )
+            job = self.update_model(job.pk, status='failed', job_explanation=msg)
+            raise RuntimeError(msg)
+
+        project_path = job.project.get_project_path(check_if_exists=False)
+        job_revision = job.project.scm_revision
+        needs_sync = True
+        if not job.project.scm_type:
+            # manual projects are not synced, user has responsibility for that
+            needs_sync = False
+        elif not os.path.exists(project_path):
+            logger.debug('Performing fresh clone of {} on this instance.'.format(job.project))
+        elif not job.project.scm_revision:
+            logger.debug('Revision not known for {}, will sync with remote'.format(job.project))
+        elif job.project.scm_type == 'git':
+            git_repo = git.Repo(project_path)
+            try:
+                desired_revision = job.project.scm_revision
+                if job.scm_branch and job.scm_branch != job.project.scm_branch:
+                    desired_revision = job.scm_branch  # could be commit or not, but will try as commit
+                current_revision = git_repo.head.commit.hexsha
+                if desired_revision == current_revision:
+                    job_revision = desired_revision
+                    logger.info('Skipping project sync for {} because commit is locally available'.format(job.log_format))
+                    needs_sync = False
+            except (ValueError, BadGitName):
+                logger.debug('Needed commit for {} not in local source tree, will sync with remote'.format(job.log_format))
+        # Galaxy requirements are not supported for manual projects
+        if not needs_sync and job.project.scm_type:
+            # see if we need a sync because of presence of roles
+            galaxy_req_path = os.path.join(project_path, 'roles', 'requirements.yml')
+            if os.path.exists(galaxy_req_path):
+                logger.debug('Running project sync for {} because of galaxy role requirements.'.format(job.log_format))
+                needs_sync = True
+
+            galaxy_collections_req_path = os.path.join(project_path, 'collections', 'requirements.yml')
+            if os.path.exists(galaxy_collections_req_path):
+                logger.debug('Running project sync for {} because of galaxy collections requirements.'.format(job.log_format))
+                needs_sync = True
+
+        if needs_sync:
             pu_ig = job.instance_group
             pu_en = job.execution_node
             if job.is_isolated() is True:
                 pu_ig = pu_ig.controller
                 pu_en = settings.CLUSTER_HOST_ID
-            if job.project.status in ('error', 'failed'):
-                msg = _(
-                    'The project revision for this job template is unknown due to a failed update.'
-                )
-                job = self.update_model(job.pk, status='failed', job_explanation=msg)
-                raise RuntimeError(msg)
-            local_project_sync = job.project.create_project_update(
-                _eager_fields=dict(
-                    launch_type="sync",
-                    job_type='run',
-                    status='running',
-                    instance_group = pu_ig,
-                    execution_node=pu_en,
-                    celery_task_id=job.celery_task_id))
+            sync_metafields = dict(
+                launch_type="sync",
+                job_type='run',
+                status='running',
+                instance_group = pu_ig,
+                execution_node=pu_en,
+                celery_task_id=job.celery_task_id
+            )
+            if job.scm_branch and job.scm_branch != job.project.scm_branch:
+                sync_metafields['scm_branch'] = job.scm_branch
+            local_project_sync = job.project.create_project_update(_eager_fields=sync_metafields)
             # save the associated job before calling run() so that a
             # cancel() call on the job can cancel the project update
             job = self.update_model(job.pk, project_update=local_project_sync)
 
             project_update_task = local_project_sync._get_task_class()
             try:
-                project_update_task().run(local_project_sync.id)
-                job = self.update_model(job.pk, scm_revision=job.project.scm_revision)
+                # the job private_data_dir is passed so sync can download roles and collections there
+                sync_task = project_update_task(job_private_data_dir=private_data_dir)
+                sync_task.run(local_project_sync.id)
+                local_project_sync.refresh_from_db()
+                job = self.update_model(job.pk, scm_revision=local_project_sync.scm_revision)
             except Exception:
                 local_project_sync.refresh_from_db()
                 if local_project_sync.status != 'canceled':
@@ -1601,6 +1688,38 @@ class RunJob(BaseTask):
                                             job_explanation=('Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' %
                                                              ('project_update', local_project_sync.name, local_project_sync.id)))
                     raise
+                job.refresh_from_db()
+                if job.cancel_flag:
+                    return
+        else:
+            # Case where a local sync is not needed, meaning that local tree is
+            # up-to-date with project, job is running project current version
+            if job_revision:
+                job = self.update_model(job.pk, scm_revision=job_revision)
+
+        # copy the project directory
+        runner_project_folder = os.path.join(private_data_dir, 'project')
+        if job.project.scm_type == 'git':
+            git_repo = git.Repo(project_path)
+            if not os.path.exists(runner_project_folder):
+                os.mkdir(runner_project_folder)
+            tmp_branch_name = 'awx_internal/{}'.format(uuid4())
+            # always clone based on specific job revision
+            if not job.scm_revision:
+                raise RuntimeError('Unexpectedly could not determine a revision to run from project.')
+            source_branch = git_repo.create_head(tmp_branch_name, job.scm_revision)
+            # git clone must take file:// syntax for source repo or else options like depth will be ignored
+            source_as_uri = Path(project_path).as_uri()
+            git.Repo.clone_from(
+                source_as_uri, runner_project_folder, branch=source_branch,
+                depth=1, single_branch=True,  # shallow, do not copy full history
+                recursive=True  # include submodules
+            )
+            # force option is necessary because remote refs are not counted, although no information is lost
+            git_repo.delete_head(tmp_branch_name, force=True)
+        else:
+            copy_tree(project_path, runner_project_folder)
+
         if job.inventory.kind == 'smart':
             # cache smart inventory memberships so that the host_filter query is not
             # ran inside of the event saving code
@@ -1637,7 +1756,24 @@ class RunProjectUpdate(BaseTask):
 
     @property
     def proot_show_paths(self):
-        return [settings.PROJECTS_ROOT]
+        show_paths = [settings.PROJECTS_ROOT]
+        if self.job_private_data_dir:
+            show_paths.append(self.job_private_data_dir)
+        return show_paths
+
+    def __init__(self, *args, job_private_data_dir=None, **kwargs):
+        super(RunProjectUpdate, self).__init__(*args, **kwargs)
+        self.playbook_new_revision = None
+        self.original_branch = None
+        self.job_private_data_dir = job_private_data_dir
+
+    def event_handler(self, event_data):
+        super(RunProjectUpdate, self).event_handler(event_data)
+        returned_data = event_data.get('event_data', {})
+        if returned_data.get('task_action', '') == 'set_fact':
+            returned_facts = returned_data.get('res', {}).get('ansible_facts', {})
+            if 'scm_version' in returned_facts:
+                self.playbook_new_revision = returned_facts['scm_version']
 
     def build_private_data(self, project_update, private_data_dir):
         '''
@@ -1652,14 +1788,17 @@ class RunProjectUpdate(BaseTask):
             }
         }
         '''
-        handle, self.revision_path = tempfile.mkstemp(dir=settings.PROJECTS_ROOT)
-        if settings.AWX_CLEANUP_PATHS:
-            self.cleanup_paths.append(self.revision_path)
         private_data = {'credentials': {}}
         if project_update.credential:
             credential = project_update.credential
             if credential.has_input('ssh_key_data'):
                 private_data['credentials'][credential] = credential.get_input('ssh_key_data', default='')
+
+        # Create dir where collections will live for the job run
+        if project_update.job_type != 'check' and getattr(self, 'job_private_data_dir'):
+            for folder_name in ('requirements_collections', 'requirements_roles'):
+                folder_path = os.path.join(self.job_private_data_dir, folder_name)
+                os.mkdir(folder_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
         return private_data
 
     def build_passwords(self, project_update, runtime_passwords):
@@ -1755,10 +1894,21 @@ class RunProjectUpdate(BaseTask):
         scm_url, extra_vars_new = self._build_scm_url_extra_vars(project_update)
         extra_vars.update(extra_vars_new)
 
-        if project_update.project.scm_revision and project_update.job_type == 'run':
+        scm_branch = project_update.scm_branch
+        branch_override = bool(project_update.scm_branch != project_update.project.scm_branch)
+        if project_update.job_type == 'run' and scm_branch and (not branch_override):
             scm_branch = project_update.project.scm_revision
+        elif not scm_branch:
+            scm_branch = {'hg': 'tip'}.get(project_update.scm_type, 'HEAD')
+        if project_update.job_type == 'check':
+            roles_enabled = False
+            collections_enabled = False
         else:
-            scm_branch = project_update.scm_branch or {'hg': 'tip'}.get(project_update.scm_type, 'HEAD')
+            roles_enabled = getattr(settings, 'AWX_ROLES_ENABLED', True)
+            collections_enabled = getattr(settings, 'AWX_COLLECTIONS_ENABLED', True)
+            # collections were introduced in Ansible version 2.8
+            if Version(get_ansible_version()) <= Version('2.8'):
+                collections_enabled = False
         extra_vars.update({
             'project_path': project_update.get_project_path(check_if_exists=False),
             'insights_url': settings.INSIGHTS_URL_BASE,
@@ -1770,17 +1920,24 @@ class RunProjectUpdate(BaseTask):
             'scm_clean': project_update.scm_clean,
             'scm_delete_on_update': project_update.scm_delete_on_update if project_update.job_type == 'check' else False,
             'scm_full_checkout': True if project_update.job_type == 'run' else False,
-            'scm_revision_output': self.revision_path,
-            'scm_revision': project_update.project.scm_revision,
-            'roles_enabled': getattr(settings, 'AWX_ROLES_ENABLED', True)
+            'roles_enabled': roles_enabled,
+            'collections_enabled': collections_enabled,
         })
+        if project_update.job_type != 'check' and self.job_private_data_dir:
+            extra_vars['collections_destination'] = os.path.join(self.job_private_data_dir, 'requirements_collections')
+            extra_vars['roles_destination'] = os.path.join(self.job_private_data_dir, 'requirements_roles')
+        # apply custom refspec from user for PR refs and the like
+        if project_update.scm_refspec:
+            extra_vars['scm_refspec'] = project_update.scm_refspec
+        elif project_update.project.allow_override:
+            # If branch is override-able, do extra fetch for all branches
+            extra_vars['scm_refspec'] = 'refs/heads/*:refs/remotes/origin/*'
         self._write_extra_vars_file(private_data_dir, extra_vars)
 
     def build_cwd(self, project_update, private_data_dir):
         return self.get_path_to('..', 'playbooks')
 
     def build_playbook_path_relative_to_cwd(self, project_update, private_data_dir):
-        self.build_cwd(project_update, private_data_dir)
         return os.path.join('project_update.yml')
 
     def get_password_prompts(self, passwords={}):
@@ -1894,25 +2051,42 @@ class RunProjectUpdate(BaseTask):
                 '{} spent {} waiting to acquire lock for local source tree '
                 'for path {}.'.format(instance.log_format, waiting_time, lock_path))
 
-    def pre_run_hook(self, instance):
+    def pre_run_hook(self, instance, private_data_dir):
         # re-create root project folder if a natural disaster has destroyed it
         if not os.path.exists(settings.PROJECTS_ROOT):
             os.mkdir(settings.PROJECTS_ROOT)
         self.acquire_lock(instance)
+        self.original_branch = None
+        if (instance.scm_type == 'git' and instance.job_type == 'run' and instance.project and
+                instance.scm_branch != instance.project.scm_branch):
+            project_path = instance.project.get_project_path(check_if_exists=False)
+            if os.path.exists(project_path):
+                git_repo = git.Repo(project_path)
+                self.original_branch = git_repo.active_branch
 
     def post_run_hook(self, instance, status):
+        if self.original_branch:
+            # for git project syncs, non-default branches can be problems
+            # restore to branch the repo was on before this run
+            try:
+                self.original_branch.checkout()
+            except Exception:
+                # this could have failed due to dirty tree, but difficult to predict all cases
+                logger.exception('Failed to restore project repo to prior state after {}'.format(instance.log_format))
         self.release_lock(instance)
         p = instance.project
+        if self.playbook_new_revision:
+            instance.scm_revision = self.playbook_new_revision
+            instance.save(update_fields=['scm_revision'])
         if instance.job_type == 'check' and status not in ('failed', 'canceled',):
-            fd = open(self.revision_path, 'r')
-            lines = fd.readlines()
-            if lines:
-                p.scm_revision = lines[0].strip()
+            if self.playbook_new_revision:
+                p.scm_revision = self.playbook_new_revision
             else:
-                logger.info("{} Could not find scm revision in check".format(instance.log_format))
+                if status == 'successful':
+                    logger.error("{} Could not find scm revision in check".format(instance.log_format))
             p.playbook_files = p.playbooks
             p.inventory_files = p.inventories
-            p.save()
+            p.save(update_fields=['scm_revision', 'playbook_files', 'inventory_files'])
 
         # Update any inventories that depend on this project
         dependent_inventory_sources = p.scm_inventory_sources.filter(update_on_project_update=True)
@@ -2133,11 +2307,12 @@ class RunInventoryUpdate(BaseTask):
         # All credentials not used by inventory source injector
         return inventory_update.get_extra_credentials()
 
-    def pre_run_hook(self, inventory_update):
+    def pre_run_hook(self, inventory_update, private_data_dir):
         source_project = None
         if inventory_update.inventory_source:
             source_project = inventory_update.inventory_source.source_project
         if (inventory_update.source=='scm' and inventory_update.launch_type!='scm' and source_project):
+            # In project sync, pulling galaxy roles is not needed
             local_project_sync = source_project.create_project_update(
                 _eager_fields=dict(
                     launch_type="sync",

awx/main/tests/data/inventory/plugins/azure_rm/env.json

@@ -1,9 +1,9 @@
 {
-    "AZURE_SUBSCRIPTION_ID": "fooo",
-    "AZURE_CLIENT_ID": "fooo",
-    "AZURE_TENANT": "fooo",
-    "AZURE_SECRET": "fooo",
-    "AZURE_CLOUD_ENVIRONMENT": "fooo",
     "ANSIBLE_JINJA2_NATIVE": "True",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
+    "AZURE_CLIENT_ID": "fooo",
+    "AZURE_CLOUD_ENVIRONMENT": "fooo",
+    "AZURE_SECRET": "fooo",
+    "AZURE_SUBSCRIPTION_ID": "fooo",
+    "AZURE_TENANT": "fooo"
 }

awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml

@@ -4,12 +4,15 @@ default_host_filters: []
 exclude_host_filters:
 - resource_group not in ['foo_resources', 'bar_resources']
 - location not in ['southcentralus', 'westus']
+fail_on_template_errors: false
 hostvar_expressions:
   ansible_host: private_ipv4_addresses[0]
   computer_name: name
-  private_ip: private_ipv4_addresses[0]
+  private_ip: private_ipv4_addresses[0] if private_ipv4_addresses else None
   provisioning_state: provisioning_state | title
-  public_ip: public_ipv4_addresses[0]
+  public_ip: public_ipv4_addresses[0] if public_ipv4_addresses else None
+  public_ip_id: public_ip_id if public_ip_id is defined else None
+  public_ip_name: public_ip_name if public_ip_name is defined else None
   tags: tags if tags else None
   type: resource_type
 keyed_groups:

awx/main/tests/data/inventory/plugins/ec2/env.json

@@ -1,7 +1,7 @@
 {
+    "ANSIBLE_JINJA2_NATIVE": "True",
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "AWS_ACCESS_KEY_ID": "fooo",
     "AWS_SECRET_ACCESS_KEY": "fooo",
-    "AWS_SECURITY_TOKEN": "fooo",
-    "ANSIBLE_JINJA2_NATIVE": "True",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "AWS_SECURITY_TOKEN": "fooo"
 }

awx/main/tests/data/inventory/plugins/gce/env.json

@@ -1,4 +1,9 @@
 {
+    "ANSIBLE_JINJA2_NATIVE": "True",
     "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
-    "GCE_CREDENTIALS_FILE_PATH": "{{ file_reference }}"
+    "GCE_CREDENTIALS_FILE_PATH": "{{ file_reference }}",
+    "GCP_AUTH_KIND": "serviceaccount",
+    "GCP_ENV_TYPE": "tower",
+    "GCP_PROJECT": "fooo",
+    "GCP_SERVICE_ACCOUNT_FILE": "{{ file_reference }}"
 }

awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml

@@ -1,6 +1,6 @@
 auth_kind: serviceaccount
 compose:
-  ansible_ssh_host: networkInterfaces[0].accessConfigs[0].natIP
+  ansible_ssh_host: networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP)
   gce_description: description if description else None
   gce_id: id
   gce_image: image
@@ -9,7 +9,7 @@ compose:
   gce_name: name
   gce_network: networkInterfaces[0].network.name
   gce_private_ip: networkInterfaces[0].networkIP
-  gce_public_ip: networkInterfaces[0].accessConfigs[0].natIP
+  gce_public_ip: networkInterfaces[0].accessConfigs[0].natIP | default(None)
   gce_status: status
   gce_subnetwork: networkInterfaces[0].subnetwork.name
   gce_tags: tags.get("items", [])

awx/main/tests/data/inventory/plugins/satellite6/env.json

@@ -1,6 +1,6 @@
 {
-    "FOREMAN_SERVER": "https://foo.invalid",
-    "FOREMAN_USER": "fooo",
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "FOREMAN_PASSWORD": "fooo",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "FOREMAN_SERVER": "https://foo.invalid",
+    "FOREMAN_USER": "fooo"
 }

awx/main/tests/data/inventory/plugins/tower/env.json

@@ -1,7 +1,7 @@
 {
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "TOWER_HOST": "https://foo.invalid",
-    "TOWER_USERNAME": "fooo",
     "TOWER_PASSWORD": "fooo",
-    "TOWER_VERIFY_SSL": "False",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "TOWER_USERNAME": "fooo",
+    "TOWER_VERIFY_SSL": "False"
 }