Bump VERSION to 6.1.0

fix reference to undefined prop for auditors

Reviewed-by: https://github.com/softwarefactory-project-zuul[bot]

.gitignore

@@ -28,6 +28,9 @@ awx/ui/build_test
 awx/ui/client/languages
 awx/ui/templates/ui/index.html
 awx/ui/templates/ui/installing.html
+awx/ui_next/node_modules/
+awx/ui_next/coverage/
+awx/ui_next/build/locales/_build
 /tower-license
 /tower-license/**
 tools/prometheus/data

Makefile

@@ -73,6 +73,9 @@ clean-ui:
 	rm -rf awx/ui/test/spec/reports/
 	rm -rf awx/ui/test/e2e/reports/
 	rm -rf awx/ui/client/languages/
+	rm -rf awx/ui_next/node_modules/
+	rm -rf awx/ui_next/coverage/
+	rm -rf awx/ui_next/build/locales/_build/
 	rm -f $(UI_DEPS_FLAG_FILE)
 	rm -f $(UI_RELEASE_DEPS_FLAG_FILE)
 	rm -f $(UI_RELEASE_FLAG_FILE)
@@ -98,7 +101,7 @@ clean: clean-ui clean-dist
 	rm -rf awx/job_status
 	rm -rf awx/job_output
 	rm -rf reports
-	rm -f awx/awx_test.sqlite3
+	rm -f awx/awx_test.sqlite3*
 	rm -rf requirements/vendor
 	rm -rf tmp
 	rm -rf $(I18N_FLAG_FILE)
@@ -124,8 +127,8 @@ virtualenv_ansible:
 		if [ ! -d "$(VENV_BASE)/ansible" ]; then \
 			virtualenv -p python --system-site-packages $(VENV_BASE)/ansible && \
 			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed six packaging appdirs && \
-			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==41.0.1 && \
-			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==19.1.1; \
+			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==36.0.1 && \
+			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==9.0.1; \
 		fi; \
 	fi
 
@@ -515,6 +518,21 @@ jshint: $(UI_DEPS_FLAG_FILE)
 # END UI TASKS
 # --------------------------------------
 
+# UI NEXT TASKS
+# --------------------------------------
+
+ui-next-lint:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next lint
+	$(NPM_BIN) run --prefix awx/ui_next prettier-check
+
+ui-next-test:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next test
+
+# END UI NEXT TASKS
+# --------------------------------------
+
 # Build a pip-installable package into dist/ with a timestamped version number.
 dev_build:
 	$(PYTHON) setup.py dev_build

VERSION

@@ -1 +1 @@
-5.0.0
+6.1.0

awx/__init__.py

@@ -25,9 +25,8 @@ import hashlib
 
 try:
     import django
-    from django.utils.encoding import force_bytes
-    from django.db.backends.base.schema import BaseDatabaseSchemaEditor
     from django.db.backends.base import schema
+    from django.db.backends.utils import names_digest
     HAS_DJANGO = True
 except ImportError:
     HAS_DJANGO = False
@@ -37,30 +36,33 @@ if HAS_DJANGO is True:
     # This line exists to make sure we don't regress on FIPS support if we
     # upgrade Django; if you're upgrading Django and see this error,
     # update the version check below, and confirm that FIPS still works.
-    if django.__version__ != '1.11.20':
-        raise RuntimeError("Django version other than 1.11.20 detected {}. \
-                Subclassing BaseDatabaseSchemaEditor is known to work for Django 1.11.20 \
-                and may not work in newer Django versions.".format(django.__version__))
+    # If operating in a FIPS environment, `hashlib.md5()` will raise a `ValueError`,
+    # but will support the `usedforsecurity` keyword on RHEL and Centos systems.
 
+    # Keep an eye on https://code.djangoproject.com/ticket/28401
+    target_version = '2.2.2'
+    if django.__version__ != target_version:
+        raise RuntimeError(
+            "Django version other than {target} detected: {current}. "
+            "Overriding `names_digest` is known to work for Django {target} "
+            "and may not work in other Django versions.".format(target=target_version,
+                                                                current=django.__version__)
+        )
 
-    class FipsBaseDatabaseSchemaEditor(BaseDatabaseSchemaEditor):
-
-        @classmethod
-        def _digest(cls, *args):
+    try:
+        names_digest('foo', 'bar', 'baz', length=8)
+    except ValueError:
+        def names_digest(*args, length):
             """
-            Generates a 32-bit digest of a set of arguments that can be used to
-            shorten identifying names.
+            Generate a 32-bit digest of a set of arguments that can be used to shorten
+            identifying names.  Support for use in FIPS environments.
             """
-            try:
-                h = hashlib.md5()
-            except ValueError:
-                h = hashlib.md5(usedforsecurity=False)
+            h = hashlib.md5(usedforsecurity=False)
             for arg in args:
-                h.update(force_bytes(arg))
-            return h.hexdigest()[:8]
+                h.update(arg.encode())
+            return h.hexdigest()[:length]
 
-
-    schema.BaseDatabaseSchemaEditor = FipsBaseDatabaseSchemaEditor
+        schema.names_digest = names_digest
 
 
 def find_commands(management_dir):

awx/api/generics.py

@@ -401,21 +401,21 @@ class ListAPIView(generics.ListAPIView, GenericAPIView):
                 continue
             if getattr(field, 'related_model', None):
                 fields.add('{}__search'.format(field.name))
-        for rel in self.model._meta.related_objects:
-            name = rel.related_name
-            if isinstance(rel, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
+        for related in self.model._meta.related_objects:
+            name = related.related_name
+            if isinstance(related, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
                 # Add underscores for polymorphic subclasses for user utility
-                name = rel.related_model._meta.verbose_name.replace(" ", "_")
+                name = related.related_model._meta.verbose_name.replace(" ", "_")
             if skip_related_name(name) or name.endswith('+'):
                 continue
             fields.add('{}__search'.format(name))
-        m2m_rel = []
-        m2m_rel += self.model._meta.local_many_to_many
+        m2m_related = []
+        m2m_related += self.model._meta.local_many_to_many
         if issubclass(self.model, UnifiedJobTemplate) and self.model != UnifiedJobTemplate:
-            m2m_rel += UnifiedJobTemplate._meta.local_many_to_many
+            m2m_related += UnifiedJobTemplate._meta.local_many_to_many
         if issubclass(self.model, UnifiedJob) and self.model != UnifiedJob:
-            m2m_rel += UnifiedJob._meta.local_many_to_many
-        for relationship in m2m_rel:
+            m2m_related += UnifiedJob._meta.local_many_to_many
+        for relationship in m2m_related:
             if skip_related_name(relationship.name):
                 continue
             if relationship.related_model._meta.app_label != 'main':

awx/api/permissions.py

@@ -95,7 +95,7 @@ class ModelAccessPermission(permissions.BasePermission):
         '''
 
         # Don't allow anonymous users. 401, not 403, hence no raised exception.
-        if not request.user or request.user.is_anonymous():
+        if not request.user or request.user.is_anonymous:
             return False
 
         # Always allow superusers

awx/api/serializers.py

@@ -1246,7 +1246,7 @@ class OrganizationSerializer(BaseSerializer):
             applications    = self.reverse('api:organization_applications_list',     kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:organization_activity_stream_list', kwargs={'pk': obj.pk}),
             notification_templates = self.reverse('api:organization_notification_templates_list', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:organization_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:organization_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:organization_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:organization_notification_templates_error_list', kwargs={'pk': obj.pk}),
             object_roles = self.reverse('api:organization_object_roles_list', kwargs={'pk': obj.pk}),
@@ -1352,7 +1352,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
             scm_inventory_sources = self.reverse('api:project_scm_inventory_sources', kwargs={'pk': obj.pk}),
             schedules = self.reverse('api:project_schedules_list', kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:project_activity_stream_list', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:project_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:project_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:project_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:project_notification_templates_error_list', kwargs={'pk': obj.pk}),
             access_list = self.reverse('api:project_access_list', kwargs={'pk': obj.pk}),
@@ -1943,6 +1943,25 @@ class InventorySourceOptionsSerializer(BaseSerializer):
 
         return super(InventorySourceOptionsSerializer, self).validate(attrs)
 
+    # TODO: remove when old 'credential' fields are removed
+    def get_summary_fields(self, obj):
+        summary_fields = super(InventorySourceOptionsSerializer, self).get_summary_fields(obj)
+        all_creds = []
+        if 'credential' in summary_fields:
+            cred = obj.get_cloud_credential()
+            if cred:
+                summarized_cred = {
+                    'id': cred.id, 'name': cred.name, 'description': cred.description,
+                    'kind': cred.kind, 'cloud': True
+                }
+                summary_fields['credential'] = summarized_cred
+                all_creds.append(summarized_cred)
+                summary_fields['credential']['credential_type_id'] = cred.credential_type_id
+            else:
+                summary_fields.pop('credential')
+        summary_fields['credentials'] = all_creds
+        return summary_fields
+
 
 class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOptionsSerializer):
 
@@ -1970,7 +1989,7 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
             activity_stream = self.reverse('api:inventory_source_activity_stream_list', kwargs={'pk': obj.pk}),
             hosts = self.reverse('api:inventory_source_hosts_list', kwargs={'pk': obj.pk}),
             groups = self.reverse('api:inventory_source_groups_list', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:inventory_source_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:inventory_source_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:inventory_source_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:inventory_source_notification_templates_error_list', kwargs={'pk': obj.pk}),
         ))
@@ -2541,7 +2560,7 @@ class CredentialSerializer(BaseSerializer):
 
     def validate_credential_type(self, credential_type):
         if self.instance and credential_type.pk != self.instance.credential_type.pk:
-            for rel in (
+            for related_objects in (
                 'ad_hoc_commands',
                 'insights_inventories',
                 'unifiedjobs',
@@ -2550,7 +2569,7 @@ class CredentialSerializer(BaseSerializer):
                 'projectupdates',
                 'workflowjobnodes'
             ):
-                if getattr(self.instance, rel).count() > 0:
+                if getattr(self.instance, related_objects).count() > 0:
                     raise ValidationError(
                         _('You cannot change the credential type of the credential, as it may break the functionality'
                           ' of the resources using it.'),
@@ -2792,7 +2811,7 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
             schedules = self.reverse('api:job_template_schedules_list', kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:job_template_activity_stream_list', kwargs={'pk': obj.pk}),
             launch = self.reverse('api:job_template_launch', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:job_template_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
             access_list = self.reverse('api:job_template_access_list',      kwargs={'pk': obj.pk}),
@@ -3204,7 +3223,7 @@ class SystemJobTemplateSerializer(UnifiedJobTemplateSerializer):
             jobs = self.reverse('api:system_job_template_jobs_list', kwargs={'pk': obj.pk}),
             schedules = self.reverse('api:system_job_template_schedules_list', kwargs={'pk': obj.pk}),
             launch = self.reverse('api:system_job_template_launch', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:system_job_template_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:system_job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:system_job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:system_job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
 
@@ -3271,7 +3290,7 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
             workflow_nodes = self.reverse('api:workflow_job_template_workflow_nodes_list', kwargs={'pk': obj.pk}),
             labels = self.reverse('api:workflow_job_template_label_list', kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:workflow_job_template_activity_stream_list', kwargs={'pk': obj.pk}),
-            notification_templates_any = self.reverse('api:workflow_job_template_notification_templates_any_list', kwargs={'pk': obj.pk}),
+            notification_templates_started = self.reverse('api:workflow_job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:workflow_job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:workflow_job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
             access_list = self.reverse('api:workflow_job_template_access_list', kwargs={'pk': obj.pk}),
@@ -4621,37 +4640,37 @@ class ActivityStreamSerializer(BaseSerializer):
             return ""
 
     def get_related(self, obj):
-        rel = {}
+        data = {}
         if obj.actor is not None:
-            rel['actor'] = self.reverse('api:user_detail', kwargs={'pk': obj.actor.pk})
+            data['actor'] = self.reverse('api:user_detail', kwargs={'pk': obj.actor.pk})
         for fk, __ in self._local_summarizable_fk_fields:
             if not hasattr(obj, fk):
                 continue
-            m2m_list = self._get_rel(obj, fk)
+            m2m_list = self._get_related_objects(obj, fk)
             if m2m_list:
-                rel[fk] = []
+                data[fk] = []
                 id_list = []
-                for thisItem in m2m_list:
-                    if getattr(thisItem, 'id', None) in id_list:
+                for item in m2m_list:
+                    if getattr(item, 'id', None) in id_list:
                         continue
-                    id_list.append(getattr(thisItem, 'id', None))
-                    if hasattr(thisItem, 'get_absolute_url'):
-                        rel_url = thisItem.get_absolute_url(self.context.get('request'))
+                    id_list.append(getattr(item, 'id', None))
+                    if hasattr(item, 'get_absolute_url'):
+                        url = item.get_absolute_url(self.context.get('request'))
                     else:
                         view_name = fk + '_detail'
-                        rel_url = self.reverse('api:' + view_name, kwargs={'pk': thisItem.id})
-                    rel[fk].append(rel_url)
+                        url = self.reverse('api:' + view_name, kwargs={'pk': item.id})
+                    data[fk].append(url)
 
                     if fk == 'schedule':
-                        rel['unified_job_template'] = thisItem.unified_job_template.get_absolute_url(self.context.get('request'))
+                        data['unified_job_template'] = item.unified_job_template.get_absolute_url(self.context.get('request'))
         if obj.setting and obj.setting.get('category', None):
-            rel['setting'] = self.reverse(
+            data['setting'] = self.reverse(
                 'api:setting_singleton_detail',
                 kwargs={'category_slug': obj.setting['category']}
             )
-        return rel
+        return data
 
-    def _get_rel(self, obj, fk):
+    def _get_related_objects(self, obj, fk):
         related_model = ActivityStream._meta.get_field(fk).related_model
         related_manager = getattr(obj, fk)
         if issubclass(related_model, PolymorphicModel) and hasattr(obj, '_prefetched_objects_cache'):
@@ -4661,43 +4680,34 @@ class ActivityStreamSerializer(BaseSerializer):
                 obj._prefetched_objects_cache[related_manager.prefetch_cache_name] = list(related_manager.all())
         return related_manager.all()
 
+    def _summarize_parent_ujt(self, obj, fk, summary_fields):
+        summary_keys = {'job': 'job_template',
+                        'workflow_job_template_node': 'workflow_job_template',
+                        'schedule': 'unified_job_template'}
+        if fk not in summary_keys:
+            return
+        related_obj = getattr(obj, summary_keys[fk], None)
+        item = {}
+        fields = SUMMARIZABLE_FK_FIELDS[summary_keys[fk]]
+        if related_obj is not None:
+            summary_fields[get_type_for_model(related_obj)] = []
+            for field in fields:
+                fval = getattr(related_obj, field, None)
+                if fval is not None:
+                    item[field] = fval
+            summary_fields[get_type_for_model(related_obj)].append(item)
+
     def get_summary_fields(self, obj):
         summary_fields = OrderedDict()
         for fk, related_fields in self._local_summarizable_fk_fields:
             try:
                 if not hasattr(obj, fk):
                     continue
-                m2m_list = self._get_rel(obj, fk)
+                m2m_list = self._get_related_objects(obj, fk)
                 if m2m_list:
                     summary_fields[fk] = []
                     for thisItem in m2m_list:
-                        if fk == 'job':
-                            summary_fields['job_template'] = []
-                            job_template_item = {}
-                            job_template_fields = SUMMARIZABLE_FK_FIELDS['job_template']
-                            job_template = getattr(thisItem, 'job_template', None)
-                            if job_template is not None:
-                                for field in job_template_fields:
-                                    fval = getattr(job_template, field, None)
-                                    if fval is not None:
-                                        job_template_item[field] = fval
-                                summary_fields['job_template'].append(job_template_item)
-                        if fk == 'workflow_job_template_node':
-                            summary_fields['workflow_job_template'] = []
-                            workflow_job_template_item = {}
-                            workflow_job_template_fields = SUMMARIZABLE_FK_FIELDS['workflow_job_template']
-                            workflow_job_template = getattr(thisItem, 'workflow_job_template', None)
-                            if workflow_job_template is not None:
-                                for field in workflow_job_template_fields:
-                                    fval = getattr(workflow_job_template, field, None)
-                                    if fval is not None:
-                                        workflow_job_template_item[field] = fval
-                                summary_fields['workflow_job_template'].append(workflow_job_template_item)
-                        if fk == 'schedule':
-                            unified_job_template = getattr(thisItem, 'unified_job_template', None)
-                            if unified_job_template is not None:
-                                summary_fields[get_type_for_model(unified_job_template)] = {'id': unified_job_template.id,
-                                                                                            'name': unified_job_template.name}
+                        self._summarize_parent_ujt(thisItem, fk, summary_fields)
                         thisItemDict = {}
                         for field in related_fields:
                             fval = getattr(thisItem, field, None)

awx/api/urls/inventory_source.py

@@ -13,8 +13,8 @@ from awx.api.views import (
     InventorySourceCredentialsList,
     InventorySourceGroupsList,
     InventorySourceHostsList,
-    InventorySourceNotificationTemplatesAnyList,
     InventorySourceNotificationTemplatesErrorList,
+    InventorySourceNotificationTemplatesStartedList,
     InventorySourceNotificationTemplatesSuccessList,
 )
 
@@ -29,8 +29,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/credentials/$', InventorySourceCredentialsList.as_view(), name='inventory_source_credentials_list'),
     url(r'^(?P<pk>[0-9]+)/groups/$', InventorySourceGroupsList.as_view(), name='inventory_source_groups_list'),
     url(r'^(?P<pk>[0-9]+)/hosts/$', InventorySourceHostsList.as_view(), name='inventory_source_hosts_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', InventorySourceNotificationTemplatesAnyList.as_view(),
-        name='inventory_source_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', InventorySourceNotificationTemplatesStartedList.as_view(),
+        name='inventory_source_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', InventorySourceNotificationTemplatesErrorList.as_view(),
         name='inventory_source_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', InventorySourceNotificationTemplatesSuccessList.as_view(),

awx/api/urls/job_template.py

@@ -13,8 +13,8 @@ from awx.api.views import (
     JobTemplateSchedulesList,
     JobTemplateSurveySpec,
     JobTemplateActivityStreamList,
-    JobTemplateNotificationTemplatesAnyList,
     JobTemplateNotificationTemplatesErrorList,
+    JobTemplateNotificationTemplatesStartedList,
     JobTemplateNotificationTemplatesSuccessList,
     JobTemplateInstanceGroupsList,
     JobTemplateAccessList,
@@ -34,8 +34,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/schedules/$', JobTemplateSchedulesList.as_view(), name='job_template_schedules_list'),
     url(r'^(?P<pk>[0-9]+)/survey_spec/$', JobTemplateSurveySpec.as_view(), name='job_template_survey_spec'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', JobTemplateActivityStreamList.as_view(), name='job_template_activity_stream_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', JobTemplateNotificationTemplatesAnyList.as_view(),
-        name='job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', JobTemplateNotificationTemplatesStartedList.as_view(),
+        name='job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', JobTemplateNotificationTemplatesErrorList.as_view(),
         name='job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', JobTemplateNotificationTemplatesSuccessList.as_view(),

awx/api/urls/organization.py

@@ -15,8 +15,8 @@ from awx.api.views import (
     OrganizationCredentialList,
     OrganizationActivityStreamList,
     OrganizationNotificationTemplatesList,
-    OrganizationNotificationTemplatesAnyList,
     OrganizationNotificationTemplatesErrorList,
+    OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
     OrganizationInstanceGroupsList,
     OrganizationObjectRolesList,
@@ -25,7 +25,7 @@ from awx.api.views import (
 )
 
 
-urls = [ 
+urls = [
     url(r'^$', OrganizationList.as_view(), name='organization_list'),
     url(r'^(?P<pk>[0-9]+)/$', OrganizationDetail.as_view(), name='organization_detail'),
     url(r'^(?P<pk>[0-9]+)/users/$', OrganizationUsersList.as_view(), name='organization_users_list'),
@@ -37,8 +37,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/credentials/$', OrganizationCredentialList.as_view(), name='organization_credential_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', OrganizationActivityStreamList.as_view(), name='organization_activity_stream_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates/$', OrganizationNotificationTemplatesList.as_view(), name='organization_notification_templates_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', OrganizationNotificationTemplatesAnyList.as_view(),
-        name='organization_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', OrganizationNotificationTemplatesStartedList.as_view(),
+        name='organization_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', OrganizationNotificationTemplatesErrorList.as_view(),
         name='organization_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', OrganizationNotificationTemplatesSuccessList.as_view(),

awx/api/urls/project.py

@@ -14,8 +14,8 @@ from awx.api.views import (
     ProjectUpdatesList,
     ProjectActivityStreamList,
     ProjectSchedulesList,
-    ProjectNotificationTemplatesAnyList,
     ProjectNotificationTemplatesErrorList,
+    ProjectNotificationTemplatesStartedList,
     ProjectNotificationTemplatesSuccessList,
     ProjectObjectRolesList,
     ProjectAccessList,
@@ -34,10 +34,11 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/project_updates/$', ProjectUpdatesList.as_view(), name='project_updates_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', ProjectActivityStreamList.as_view(), name='project_activity_stream_list'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', ProjectSchedulesList.as_view(), name='project_schedules_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', ProjectNotificationTemplatesAnyList.as_view(), name='project_notification_templates_any_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', ProjectNotificationTemplatesErrorList.as_view(), name='project_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', ProjectNotificationTemplatesSuccessList.as_view(),
         name='project_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', ProjectNotificationTemplatesStartedList.as_view(),
+        name='project_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', ProjectObjectRolesList.as_view(), name='project_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', ProjectAccessList.as_view(), name='project_access_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', ProjectCopy.as_view(), name='project_copy'),

awx/api/urls/system_job_template.py

@@ -9,8 +9,8 @@ from awx.api.views import (
     SystemJobTemplateLaunch,
     SystemJobTemplateJobsList,
     SystemJobTemplateSchedulesList,
-    SystemJobTemplateNotificationTemplatesAnyList,
     SystemJobTemplateNotificationTemplatesErrorList,
+    SystemJobTemplateNotificationTemplatesStartedList,
     SystemJobTemplateNotificationTemplatesSuccessList,
 )
 
@@ -21,8 +21,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/launch/$', SystemJobTemplateLaunch.as_view(), name='system_job_template_launch'),
     url(r'^(?P<pk>[0-9]+)/jobs/$', SystemJobTemplateJobsList.as_view(), name='system_job_template_jobs_list'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', SystemJobTemplateSchedulesList.as_view(), name='system_job_template_schedules_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', SystemJobTemplateNotificationTemplatesAnyList.as_view(),
-        name='system_job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', SystemJobTemplateNotificationTemplatesStartedList.as_view(),
+        name='system_job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', SystemJobTemplateNotificationTemplatesErrorList.as_view(),
         name='system_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', SystemJobTemplateNotificationTemplatesSuccessList.as_view(),

awx/api/urls/workflow_job_template.py

@@ -13,8 +13,8 @@ from awx.api.views import (
     WorkflowJobTemplateSurveySpec,
     WorkflowJobTemplateWorkflowNodesList,
     WorkflowJobTemplateActivityStreamList,
-    WorkflowJobTemplateNotificationTemplatesAnyList,
     WorkflowJobTemplateNotificationTemplatesErrorList,
+    WorkflowJobTemplateNotificationTemplatesStartedList,
     WorkflowJobTemplateNotificationTemplatesSuccessList,
     WorkflowJobTemplateAccessList,
     WorkflowJobTemplateObjectRolesList,
@@ -32,8 +32,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/survey_spec/$', WorkflowJobTemplateSurveySpec.as_view(), name='workflow_job_template_survey_spec'),
     url(r'^(?P<pk>[0-9]+)/workflow_nodes/$', WorkflowJobTemplateWorkflowNodesList.as_view(), name='workflow_job_template_workflow_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', WorkflowJobTemplateActivityStreamList.as_view(), name='workflow_job_template_activity_stream_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', WorkflowJobTemplateNotificationTemplatesAnyList.as_view(),
-        name='workflow_job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', WorkflowJobTemplateNotificationTemplatesStartedList.as_view(),
+        name='workflow_job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', WorkflowJobTemplateNotificationTemplatesErrorList.as_view(),
         name='workflow_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', WorkflowJobTemplateNotificationTemplatesSuccessList.as_view(),

awx/api/views/__init__.py

@@ -116,6 +116,7 @@ from awx.api.views.organization import ( # noqa
     OrganizationNotificationTemplatesList,
     OrganizationNotificationTemplatesAnyList,
     OrganizationNotificationTemplatesErrorList,
+    OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
     OrganizationInstanceGroupsList,
     OrganizationAccessList,
@@ -747,22 +748,20 @@ class ProjectNotificationTemplatesAnyList(SubListCreateAttachDetachAPIView):
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.Project
-    relationship = 'notification_templates_any'
 
 
-class ProjectNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class ProjectNotificationTemplatesStartedList(ProjectNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class ProjectNotificationTemplatesErrorList(ProjectNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.Project
     relationship = 'notification_templates_error'
 
 
-class ProjectNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class ProjectNotificationTemplatesSuccessList(ProjectNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.Project
     relationship = 'notification_templates_success'
 
 
@@ -2102,7 +2101,6 @@ class InventorySourceNotificationTemplatesAnyList(SubListCreateAttachDetachAPIVi
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.InventorySource
-    relationship = 'notification_templates_any'
 
     def post(self, request, *args, **kwargs):
         parent = self.get_parent_object()
@@ -2113,6 +2111,11 @@ class InventorySourceNotificationTemplatesAnyList(SubListCreateAttachDetachAPIVi
         return super(InventorySourceNotificationTemplatesAnyList, self).post(request, *args, **kwargs)
 
 
+class InventorySourceNotificationTemplatesStartedList(InventorySourceNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
 class InventorySourceNotificationTemplatesErrorList(InventorySourceNotificationTemplatesAnyList):
 
     relationship = 'notification_templates_error'
@@ -2626,22 +2629,20 @@ class JobTemplateNotificationTemplatesAnyList(SubListCreateAttachDetachAPIView):
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.JobTemplate
-    relationship = 'notification_templates_any'
 
 
-class JobTemplateNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class JobTemplateNotificationTemplatesStartedList(JobTemplateNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class JobTemplateNotificationTemplatesErrorList(JobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.JobTemplate
     relationship = 'notification_templates_error'
 
 
-class JobTemplateNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class JobTemplateNotificationTemplatesSuccessList(JobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.JobTemplate
     relationship = 'notification_templates_success'
 
 
@@ -2996,7 +2997,7 @@ class WorkflowJobTemplateNodeChildrenBaseList(EnforceParentRelationshipMixin, Su
         relationships = ['success_nodes', 'failure_nodes', 'always_nodes']
         relationships.remove(self.relationship)
         qs = functools.reduce(lambda x, y: (x | y),
-                              (Q(**{'{}__in'.format(rel): [sub.id]}) for rel in relationships))
+                              (Q(**{'{}__in'.format(r): [sub.id]}) for r in relationships))
 
         if models.WorkflowJobTemplateNode.objects.filter(Q(pk=parent.id) & qs).exists():
             return {"Error": _("Relationship not allowed.")}
@@ -3234,22 +3235,20 @@ class WorkflowJobTemplateNotificationTemplatesAnyList(SubListCreateAttachDetachA
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.WorkflowJobTemplate
-    relationship = 'notification_templates_any'
 
 
-class WorkflowJobTemplateNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class WorkflowJobTemplateNotificationTemplatesStartedList(WorkflowJobTemplateNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class WorkflowJobTemplateNotificationTemplatesErrorList(WorkflowJobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.WorkflowJobTemplate
     relationship = 'notification_templates_error'
 
 
-class WorkflowJobTemplateNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class WorkflowJobTemplateNotificationTemplatesSuccessList(WorkflowJobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.WorkflowJobTemplate
     relationship = 'notification_templates_success'
 
 
@@ -3411,22 +3410,20 @@ class SystemJobTemplateNotificationTemplatesAnyList(SubListCreateAttachDetachAPI
     model = models.NotificationTemplate
     serializer_class = serializers.NotificationTemplateSerializer
     parent_model = models.SystemJobTemplate
-    relationship = 'notification_templates_any'
 
 
-class SystemJobTemplateNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class SystemJobTemplateNotificationTemplatesStartedList(SystemJobTemplateNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class SystemJobTemplateNotificationTemplatesErrorList(SystemJobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.SystemJobTemplate
     relationship = 'notification_templates_error'
 
 
-class SystemJobTemplateNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class SystemJobTemplateNotificationTemplatesSuccessList(SystemJobTemplateNotificationTemplatesAnyList):
 
-    model = models.NotificationTemplate
-    serializer_class = serializers.NotificationTemplateSerializer
-    parent_model = models.SystemJobTemplate
     relationship = 'notification_templates_success'
 
 

awx/api/views/organization.py

@@ -178,22 +178,20 @@ class OrganizationNotificationTemplatesAnyList(SubListCreateAttachDetachAPIView)
     model = NotificationTemplate
     serializer_class = NotificationTemplateSerializer
     parent_model = Organization
-    relationship = 'notification_templates_any'
 
 
-class OrganizationNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class OrganizationNotificationTemplatesStartedList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class OrganizationNotificationTemplatesErrorList(OrganizationNotificationTemplatesAnyList):
 
-    model = NotificationTemplate
-    serializer_class = NotificationTemplateSerializer
-    parent_model = Organization
     relationship = 'notification_templates_error'
 
 
-class OrganizationNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class OrganizationNotificationTemplatesSuccessList(OrganizationNotificationTemplatesAnyList):
 
-    model = NotificationTemplate
-    serializer_class = NotificationTemplateSerializer
-    parent_model = Organization
     relationship = 'notification_templates_success'
 
 

awx/conf/migrations/0001_initial.py

@@ -21,7 +21,8 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('key', models.CharField(max_length=255)),
                 ('value', jsonfield.fields.JSONField(null=True)),
-                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('user', models.ForeignKey(related_name='settings', default=None, editable=False,
+                                           to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True)),
             ],
             options={
                 'abstract': False,

awx/conf/tests/functional/test_api.py

@@ -8,6 +8,7 @@ from awx.main.utils.encryption import decrypt_field
 from awx.conf import fields
 from awx.conf.registry import settings_registry
 from awx.conf.models import Setting
+from awx.sso import fields as sso_fields
 
 
 @pytest.fixture
@@ -137,7 +138,7 @@ def test_setting_signleton_retrieve_hierachy(api_request, dummy_setting):
 
 
 @pytest.mark.django_db
-def test_setting_signleton_retrieve_readonly(api_request, dummy_setting):
+def test_setting_singleton_retrieve_readonly(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.IntegerField,
@@ -183,6 +184,30 @@ def test_setting_singleton_update(api_request, dummy_setting):
         assert response.data['FOO_BAR'] == 4
 
 
+@pytest.mark.django_db
+def test_setting_singleton_update_hybriddictfield_with_forbidden(api_request, dummy_setting):
+    # Some HybridDictField subclasses have a child of _Forbidden,
+    # indicating that only the defined fields can be filled in.  Make
+    # sure that the _Forbidden validator doesn't get used for the
+    # fields.  See also https://github.com/ansible/awx/issues/4099.
+    with dummy_setting(
+        'FOO_BAR',
+        field_class=sso_fields.SAMLOrgAttrField,
+        category='FooBar',
+        category_slug='foobar',
+    ), mock.patch('awx.conf.views.handle_setting_changes'):
+        api_request(
+            'patch',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}),
+            data={'FOO_BAR': {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}}
+        )
+        response = api_request(
+            'get',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'})
+        )
+        assert response.data['FOO_BAR'] == {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}
+
+
 @pytest.mark.django_db
 def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy_setting):
     with dummy_setting(
@@ -206,7 +231,7 @@ def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy
 
 
 @pytest.mark.django_db
-def test_setting_singleton_update_dont_change_encripted_mark(api_request, dummy_setting):
+def test_setting_singleton_update_dont_change_encrypted_mark(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.CharField,

awx/main/access.py

@@ -1265,7 +1265,7 @@ class TeamAccess(BaseAccess):
                 (self.user.admin_of_organizations.exists() or self.user.auditor_of_organizations.exists()):
             return self.model.objects.all()
         return self.model.objects.filter(
-            Q(organization=Organization.accessible_pk_qs(self.user, 'member_role')) |
+            Q(organization__in=Organization.accessible_pk_qs(self.user, 'member_role')) |
             Q(pk__in=self.model.accessible_pk_qs(self.user, 'read_role'))
         )
 

awx/main/analytics/collectors.py

@@ -153,10 +153,16 @@ def projects_by_scm_type(since):
     return counts
 
 
+def _get_isolated_datetime(last_check):
+    if last_check:
+        return last_check.isoformat()
+    return last_check
+
+
 @register('instance_info')
 def instance_info(since):
     info = {}
-    instances = models.Instance.objects.values_list('hostname').annotate().values(
+    instances = models.Instance.objects.values_list('hostname').values(
         'uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'hostname', 'last_isolated_check', 'enabled')
     for instance in instances:
         instance_info = {
@@ -166,7 +172,7 @@ def instance_info(since):
             'cpu': instance['cpu'],
             'memory': instance['memory'],
             'managed_by_policy': instance['managed_by_policy'],
-            'last_isolated_check': instance['last_isolated_check'],
+            'last_isolated_check': _get_isolated_datetime(instance['last_isolated_check']),
             'enabled': instance['enabled']
         }
         info[instance['uuid']] = instance_info
@@ -187,12 +193,12 @@ def job_counts(since):
 def job_instance_counts(since):
     counts = {}
     job_types = models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
-        'execution_node', 'launch_type').annotate(job_launch_type=Count('launch_type'))
+        'execution_node', 'launch_type').annotate(job_launch_type=Count('launch_type')).order_by()
     for job in job_types:
         counts.setdefault(job[0], {}).setdefault('launch_type', {})[job[1]] = job[2]
         
     job_statuses = models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
-        'execution_node', 'status').annotate(job_status=Count('status'))
+        'execution_node', 'status').annotate(job_status=Count('status')).order_by()
     for job in job_statuses:
         counts.setdefault(job[0], {}).setdefault('status', {})[job[1]] = job[2]
     return counts

awx/main/analytics/core.py

@@ -119,24 +119,28 @@ def ship(path):
     """
     Ship gathered metrics via the Insights agent
     """
-    agent = 'insights-client'
-    if shutil.which(agent) is None:
-        logger.error('could not find {} on PATH'.format(agent))
-        return
-    logger.debug('shipping analytics file: {}'.format(path))
     try:
-        cmd = [
-            agent, '--payload', path, '--content-type', settings.INSIGHTS_AGENT_MIME
-        ]
-        output = smart_str(subprocess.check_output(cmd, timeout=60 * 5))
-        logger.debug(output)
-        # reset the `last_run` when data is shipped
-        run_now = now()
-        state = TowerAnalyticsState.get_solo()
-        state.last_run = run_now
-        state.save()
+        agent = 'insights-client'
+        if shutil.which(agent) is None:
+            logger.error('could not find {} on PATH'.format(agent))
+            return
+        logger.debug('shipping analytics file: {}'.format(path))
+        try:
+            cmd = [
+                agent, '--payload', path, '--content-type', settings.INSIGHTS_AGENT_MIME
+            ]
+            output = smart_str(subprocess.check_output(cmd, timeout=60 * 5))
+            logger.debug(output)
+            # reset the `last_run` when data is shipped
+            run_now = now()
+            state = TowerAnalyticsState.get_solo()
+            state.last_run = run_now
+            state.save()
 
-    except subprocess.CalledProcessError:
-        logger.exception('{} failure:'.format(cmd))
-    except subprocess.TimeoutExpired:
-        logger.exception('{} timeout:'.format(cmd))
+        except subprocess.CalledProcessError:
+            logger.exception('{} failure:'.format(cmd))
+        except subprocess.TimeoutExpired:
+            logger.exception('{} timeout:'.format(cmd))
+    finally:
+        # cleanup tar.gz
+        os.remove(path)

awx/main/consumers.py

@@ -24,7 +24,7 @@ def ws_connect(message):
     headers = dict(message.content.get('headers', ''))
     message.reply_channel.send({"accept": True})
     message.content['method'] = 'FAKE'
-    if message.user.is_authenticated():
+    if message.user.is_authenticated:
         message.reply_channel.send(
             {"text": json.dumps({"accept": True, "user": message.user.id})}
         )

awx/main/credential_plugins/aim.py

@@ -1,14 +1,14 @@
 from .plugin import CredentialPlugin
 
-import os
-import stat
-import tempfile
-import threading
 from urllib.parse import quote, urlencode, urljoin
 
 from django.utils.translation import ugettext_lazy as _
 import requests
 
+# AWX
+from awx.main.utils import (
+    create_temporary_fifo,
+)
 
 aim_inputs = {
     'fields': [{
@@ -60,24 +60,6 @@ aim_inputs = {
 }
 
 
-def create_temporary_fifo(data):
-    """Open fifo named pipe in a new thread using a temporary file path. The
-    thread blocks until data is read from the pipe.
-
-    Returns the path to the fifo.
-
-    :param data(bytes): Data to write to the pipe.
-    """
-    path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names()))
-    os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR)
-
-    threading.Thread(
-        target=lambda p, d: open(p, 'wb').write(d),
-        args=(path, data)
-    ).start()
-    return path
-
-
 def aim_backend(**kwargs):
     url = kwargs['url']
     client_cert = kwargs.get('client_cert', None)

awx/main/credential_plugins/conjur.py

@@ -1,15 +1,16 @@
 from .plugin import CredentialPlugin
 
 import base64
-import os
-import stat
-import tempfile
-import threading
 from urllib.parse import urljoin, quote_plus
 
 from django.utils.translation import ugettext_lazy as _
 import requests
 
+# AWX
+from awx.main.utils import (
+    create_temporary_fifo,
+)
+
 
 conjur_inputs = {
     'fields': [{
@@ -51,24 +52,6 @@ conjur_inputs = {
 }
 
 
-def create_temporary_fifo(data):
-    """Open fifo named pipe in a new thread using a temporary file path. The
-    thread blocks until data is read from the pipe.
-
-    Returns the path to the fifo.
-
-    :param data(bytes): Data to write to the pipe.
-    """
-    path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names()))
-    os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR)
-
-    threading.Thread(
-        target=lambda p, d: open(p, 'wb').write(d),
-        args=(path, data)
-    ).start()
-    return path
-
-
 def conjur_backend(**kwargs):
     url = kwargs['url']
     api_key = kwargs['api_key']

awx/main/credential_plugins/hashivault.py

@@ -8,6 +8,10 @@ from .plugin import CredentialPlugin
 import requests
 from django.utils.translation import ugettext_lazy as _
 
+# AWX
+from awx.main.utils import (
+    create_temporary_fifo,
+)
 
 base_inputs = {
     'fields': [{
@@ -22,12 +26,18 @@ base_inputs = {
         'type': 'string',
         'secret': True,
         'help_text': _('The access token used to authenticate to the Vault server'),
+    }, {
+        'id': 'cacert',
+        'label': _('CA Certificate'),
+        'type': 'string',
+        'multiline': True,
+        'help_text': _('The CA certificate used to verify the SSL certificate of the Vault server')
     }],
     'metadata': [{
         'id': 'secret_path',
         'label': _('Path to Secret'),
         'type': 'string',
-        'help_text': _('The path to the secret e.g., /some-engine/some-secret/'),
+        'help_text': _('The path to the secret stored in the secret backend e.g, /some/secret/')
     }],
     'required': ['url', 'token', 'secret_path'],
 }
@@ -40,7 +50,12 @@ hashi_kv_inputs['fields'].append({
     'help_text': _('API v1 is for static key/value lookups.  API v2 is for versioned key/value lookups.'),
     'default': 'v1',
 })
-hashi_kv_inputs['metadata'].extend([{
+hashi_kv_inputs['metadata'] = [{
+    'id': 'secret_backend',
+    'label': _('Name of Secret Backend'),
+    'type': 'string',
+    'help_text': _('The name of the kv secret backend (if left empty, the first segment of the secret path will be used).')
+}] + hashi_kv_inputs['metadata'] + [{
     'id': 'secret_key',
     'label': _('Key Name'),
     'type': 'string',
@@ -50,7 +65,7 @@ hashi_kv_inputs['metadata'].extend([{
     'label': _('Secret Version (v2 only)'),
     'type': 'string',
     'help_text': _('Used to specify a specific secret version (if left empty, the latest version will be used).'),
-}])
+}]
 hashi_kv_inputs['required'].extend(['api_version', 'secret_key'])
 
 hashi_ssh_inputs = copy.deepcopy(base_inputs)
@@ -75,36 +90,46 @@ hashi_ssh_inputs['required'].extend(['public_key', 'role'])
 
 def kv_backend(**kwargs):
     token = kwargs['token']
-    url = urljoin(kwargs['url'], 'v1')
+    url = kwargs['url']
     secret_path = kwargs['secret_path']
+    secret_backend = kwargs.get('secret_backend', None)
     secret_key = kwargs.get('secret_key', None)
-
+    cacert = kwargs.get('cacert', None)
     api_version = kwargs['api_version']
 
+    request_kwargs = {'timeout': 30}
+    if cacert:
+        request_kwargs['verify'] = create_temporary_fifo(cacert.encode())
+
     sess = requests.Session()
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
+
     if api_version == 'v2':
-        params = {}
         if kwargs.get('secret_version'):
-            params['version'] = kwargs['secret_version']
-        try:
-            mount_point, *path = pathlib.Path(secret_path.lstrip(os.sep)).parts
-            '/'.join(*path)
-        except Exception:
-            mount_point, path = secret_path, []
-        # https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version
-        response = sess.get(
-            '/'.join([url, mount_point, 'data'] + path).rstrip('/'),
-            params=params,
-            timeout=30
-        )
-        response.raise_for_status()
-        json = response.json()['data']
+            request_kwargs['params'] = {'version': kwargs['secret_version']}
+        if secret_backend:
+            path_segments = [secret_backend, 'data', secret_path]
+        else:
+            try:
+                mount_point, *path = pathlib.Path(secret_path.lstrip(os.sep)).parts
+                '/'.join(path)
+            except Exception:
+                mount_point, path = secret_path, []
+            # https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version
+            path_segments = [mount_point, 'data'] + path
     else:
-        # https://www.vaultproject.io/api/secret/kv/kv-v1.html#read-secret
-        response = sess.get('/'.join([url, secret_path]).rstrip('/'), timeout=30)
-        response.raise_for_status()
-        json = response.json()
+        if secret_backend:
+            path_segments = [secret_backend, secret_path]
+        else:
+            path_segments = [secret_path]
+
+    request_url = urljoin(url, '/'.join(['v1'] + path_segments)).rstrip('/')
+    response = sess.get(request_url, **request_kwargs)
+    response.raise_for_status()
+
+    json = response.json()
+    if api_version == 'v2':
+        json = json['data']
 
     if secret_key:
         try:
@@ -121,20 +146,22 @@ def ssh_backend(**kwargs):
     url = urljoin(kwargs['url'], 'v1')
     secret_path = kwargs['secret_path']
     role = kwargs['role']
+    cacert = kwargs.get('cacert', None)
+
+    request_kwargs = {'timeout': 30}
+    if cacert:
+        request_kwargs['verify'] = create_temporary_fifo(cacert.encode())
+
+    request_kwargs['json'] = {'public_key': kwargs['public_key']}
+    if kwargs.get('valid_principals'):
+        request_kwargs['json']['valid_principals'] = kwargs['valid_principals']
 
     sess = requests.Session()
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
-    json = {
-        'public_key': kwargs['public_key']
-    }
-    if kwargs.get('valid_principals'):
-        json['valid_principals'] = kwargs['valid_principals']
     # https://www.vaultproject.io/api/secret/ssh/index.html#sign-ssh-key
-    resp = sess.post(
-        '/'.join([url, secret_path, 'sign', role]).rstrip('/'),
-        json=json,
-        timeout=30
-    )
+    request_url = '/'.join([url, secret_path, 'sign', role]).rstrip('/')
+    resp = sess.post(request_url, **request_kwargs)
+
     resp.raise_for_status()
     return resp.json()['data']['signed_key']
 

awx/main/fields.py

@@ -11,8 +11,9 @@ from jinja2 import Environment, StrictUndefined
 from jinja2.exceptions import UndefinedError, TemplateSyntaxError
 
 # Django
-import django
+from django.contrib.postgres.fields import JSONField as upstream_JSONBField
 from django.core import exceptions as django_exceptions
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db.models.signals import (
     post_save,
     post_delete,
@@ -37,7 +38,6 @@ import jsonschema.exceptions
 
 # Django-JSONField
 from jsonfield import JSONField as upstream_JSONField
-from jsonbfield.fields import JSONField as upstream_JSONBField
 
 # DRF
 from rest_framework import serializers
@@ -76,10 +76,10 @@ class JSONField(upstream_JSONField):
     def db_type(self, connection):
         return 'text'
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         if value in {'', None} and not self.null:
             return {}
-        return super(JSONField, self).from_db_value(value, expression, connection, context)
+        return super(JSONField, self).from_db_value(value, expression, connection)
 
 
 class JSONBField(upstream_JSONBField):
@@ -91,12 +91,12 @@ class JSONBField(upstream_JSONBField):
     def get_db_prep_value(self, value, connection, prepared=False):
         if connection.vendor == 'sqlite':
             # sqlite (which we use for tests) does not support jsonb;
-            return json.dumps(value)
+            return json.dumps(value, cls=DjangoJSONEncoder)
         return super(JSONBField, self).get_db_prep_value(
             value, connection, prepared
         )
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         # Work around a bug in django-jsonfield
         # https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos
         if isinstance(value, str):
@@ -112,14 +112,9 @@ class AutoSingleRelatedObjectDescriptor(ReverseOneToOneDescriptor):
 
     def __get__(self, instance, instance_type=None):
         try:
-            return super(AutoSingleRelatedObjectDescriptor,
-                         self).__get__(instance, instance_type)
+            return super(AutoSingleRelatedObjectDescriptor, self).__get__(instance, instance_type)
         except self.related.related_model.DoesNotExist:
             obj = self.related.related_model(**{self.related.field.name: instance})
-            if self.related.field.rel.parent_link:
-                raise NotImplementedError('not supported with polymorphic!')
-                for f in instance._meta.local_fields:
-                    setattr(obj, f.name, getattr(instance, f.name))
             obj.save()
             return obj
 
@@ -453,21 +448,6 @@ class JSONSchemaField(JSONBField):
                 params={'value': value},
             )
 
-    def get_db_prep_value(self, value, connection, prepared=False):
-        if connection.vendor == 'sqlite':
-            # sqlite (which we use for tests) does not support jsonb;
-            return json.dumps(value)
-        return super(JSONSchemaField, self).get_db_prep_value(
-            value, connection, prepared
-        )
-
-    def from_db_value(self, value, expression, connection, context):
-        # Work around a bug in django-jsonfield
-        # https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos
-        if isinstance(value, str):
-            return json.loads(value)
-        return value
-
 
 @JSONSchemaField.format_checker.checks('vault_id')
 def format_vault_id(value):
@@ -986,7 +966,7 @@ class OAuth2ClientSecretField(models.CharField):
             encrypt_value(value), connection, prepared
         )
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         if value and value.startswith('$encrypted$'):
             return decrypt_value(get_encryption_key('value', pk=None), value)
         return value
@@ -1022,38 +1002,6 @@ class OrderedManyToManyDescriptor(ManyToManyDescriptor):
                         '%s__position' % self.through._meta.model_name
                     )
 
-                def add(self, *objs):
-                    # Django < 2 doesn't support this method on
-                    # ManyToManyFields w/ an intermediary model
-                    # We should be able to remove this code snippet when we
-                    # upgrade Django.
-                    # see: https://github.com/django/django/blob/stable/1.11.x/django/db/models/fields/related_descriptors.py#L926
-                    if not django.__version__.startswith('1.'):
-                        raise RuntimeError(
-                            'This method is no longer necessary in Django>=2'
-                        )
-                    try:
-                        self.through._meta.auto_created = True
-                        super(OrderedManyRelatedManager, self).add(*objs)
-                    finally:
-                        self.through._meta.auto_created = False
-
-                def remove(self, *objs):
-                    # Django < 2 doesn't support this method on
-                    # ManyToManyFields w/ an intermediary model
-                    # We should be able to remove this code snippet when we
-                    # upgrade Django.
-                    # see: https://github.com/django/django/blob/stable/1.11.x/django/db/models/fields/related_descriptors.py#L944
-                    if not django.__version__.startswith('1.'):
-                        raise RuntimeError(
-                            'This method is no longer necessary in Django>=2'
-                        )
-                    try:
-                        self.through._meta.auto_created = True
-                        super(OrderedManyRelatedManager, self).remove(*objs)
-                    finally:
-                        self.through._meta.auto_created = False
-
             return OrderedManyRelatedManager
 
         return add_custom_queryset_to_many_related_manager(

awx/main/management/commands/run_dispatcher.py

@@ -10,6 +10,7 @@ from django.core.management.base import BaseCommand
 from django.db import connection as django_connection, connections
 from kombu import Exchange, Queue
 
+from awx.main.utils.handlers import AWXProxyHandler
 from awx.main.dispatch import get_local_queuename, reaper
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.kombu import Connection
@@ -121,6 +122,12 @@ class Command(BaseCommand):
 
         reaper.reap()
         consumer = None
+
+        # don't ship external logs inside the dispatcher's parent process
+        # this exists to work around a race condition + deadlock bug on fork
+        # in cpython itself:
+        # https://bugs.python.org/issue37429
+        AWXProxyHandler.disable()
         with Connection(settings.BROKER_URL) as conn:
             try:
                 bcast = 'tower_broadcast_all'

awx/main/middleware.py

@@ -73,7 +73,7 @@ class ActivityStreamMiddleware(threading.local, MiddlewareMixin):
         super().__init__(get_response)
 
     def process_request(self, request):
-        if hasattr(request, 'user') and hasattr(request.user, 'is_authenticated') and request.user.is_authenticated():
+        if hasattr(request, 'user') and request.user.is_authenticated:
             user = request.user
         else:
             user = None

awx/main/migrations/0001_initial.py

@@ -44,7 +44,7 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('host_name', models.CharField(default='', max_length=1024, editable=False)),
                 ('event', models.CharField(max_length=100, choices=[('runner_on_failed', 'Host Failed'), ('runner_on_ok', 'Host OK'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_skipped', 'Host Skipped')])),
-                ('event_data', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('counter', models.PositiveIntegerField(default=0)),
@@ -62,7 +62,7 @@ class Migration(migrations.Migration):
                 ('expires', models.DateTimeField(default=django.utils.timezone.now)),
                 ('request_hash', models.CharField(default='', max_length=40, blank=True)),
                 ('reason', models.CharField(default='', help_text='Reason the auth token was invalidated.', max_length=1024, blank=True)),
-                ('user', models.ForeignKey(related_name='auth_tokens', to=settings.AUTH_USER_MODEL)),
+                ('user', models.ForeignKey(related_name='auth_tokens', on_delete=models.CASCADE, to=settings.AUTH_USER_MODEL)),
             ],
         ),
         migrations.CreateModel(
@@ -198,7 +198,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('event', models.CharField(max_length=100, choices=[('runner_on_failed', 'Host Failed'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete')])),
-                ('event_data', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('host_name', models.CharField(default='', max_length=1024, editable=False)),
@@ -241,7 +241,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('created', models.DateTimeField(auto_now_add=True)),
                 ('modified', models.DateTimeField(auto_now=True)),
-                ('instance', models.ForeignKey(to='main.Instance')),
+                ('instance', models.ForeignKey(on_delete=models.CASCADE, to='main.Instance')),
             ],
         ),
         migrations.CreateModel(
@@ -287,7 +287,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('ldap_dn', models.CharField(default='', max_length=1024)),
-                ('user', awx.main.fields.AutoOneToOneField(related_name='profile', editable=False, to=settings.AUTH_USER_MODEL)),
+                ('user', awx.main.fields.AutoOneToOneField(related_name='profile', editable=False, on_delete=models.CASCADE, to=settings.AUTH_USER_MODEL)),
             ],
         ),
         migrations.CreateModel(
@@ -304,7 +304,7 @@ class Migration(migrations.Migration):
                 ('dtend', models.DateTimeField(default=None, null=True, editable=False)),
                 ('rrule', models.CharField(max_length=255)),
                 ('next_run', models.DateTimeField(default=None, null=True, editable=False)),
-                ('extra_data', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('extra_data', jsonfield.fields.JSONField(default=dict, blank=True)),
                 ('created_by', models.ForeignKey(related_name="{u'class': 'schedule', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('modified_by', models.ForeignKey(related_name="{u'class': 'schedule', u'app_label': 'main'}(class)s_modified+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')),
@@ -343,7 +343,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=512)),
                 ('old_pk', models.PositiveIntegerField(default=None, null=True, editable=False)),
                 ('launch_type', models.CharField(default='manual', max_length=20, editable=False, choices=[('manual', 'Manual'), ('relaunch', 'Relaunch'), ('callback', 'Callback'), ('scheduled', 'Scheduled'), ('dependency', 'Dependency')])),
-                ('cancel_flag', models.BooleanField(default=False, editable=False)),
+                ('cancel_flag', models.BooleanField(blank=True, default=False, editable=False)),
                 ('status', models.CharField(default='new', max_length=20, editable=False, choices=[('new', 'New'), ('pending', 'Pending'), ('waiting', 'Waiting'), ('running', 'Running'), ('successful', 'Successful'), ('failed', 'Failed'), ('error', 'Error'), ('canceled', 'Canceled')])),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('started', models.DateTimeField(default=None, null=True, editable=False)),
@@ -351,7 +351,7 @@ class Migration(migrations.Migration):
                 ('elapsed', models.DecimalField(editable=False, max_digits=12, decimal_places=3)),
                 ('job_args', models.TextField(default='', editable=False, blank=True)),
                 ('job_cwd', models.CharField(default='', max_length=1024, editable=False, blank=True)),
-                ('job_env', jsonfield.fields.JSONField(default={}, editable=False, blank=True)),
+                ('job_env', jsonfield.fields.JSONField(default=dict, editable=False, blank=True)),
                 ('job_explanation', models.TextField(default='', editable=False, blank=True)),
                 ('start_args', models.TextField(default='', editable=False, blank=True)),
                 ('result_stdout_text', models.TextField(default='', editable=False, blank=True)),
@@ -380,7 +380,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='AdHocCommand',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('job_type', models.CharField(default='run', max_length=64, choices=[('run', 'Run'), ('check', 'Check')])),
                 ('limit', models.CharField(default='', max_length=1024, blank=True)),
                 ('module_name', models.CharField(default='', max_length=1024, blank=True)),
@@ -394,7 +394,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='InventorySource',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('source', models.CharField(default='', max_length=32, blank=True, choices=[('', 'Manual'), ('file', 'Local File, Directory or Script'), ('rax', 'Rackspace Cloud Servers'), ('ec2', 'Amazon EC2'), ('gce', 'Google Compute Engine'), ('azure', 'Microsoft Azure'), ('vmware', 'VMware vCenter'), ('openstack', 'OpenStack'), ('custom', 'Custom Script')])),
                 ('source_path', models.CharField(default='', max_length=1024, editable=False, blank=True)),
                 ('source_vars', models.TextField(default='', help_text='Inventory source variables in YAML or JSON format.', blank=True)),
@@ -411,7 +411,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='InventoryUpdate',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('source', models.CharField(default='', max_length=32, blank=True, choices=[('', 'Manual'), ('file', 'Local File, Directory or Script'), ('rax', 'Rackspace Cloud Servers'), ('ec2', 'Amazon EC2'), ('gce', 'Google Compute Engine'), ('azure', 'Microsoft Azure'), ('vmware', 'VMware vCenter'), ('openstack', 'OpenStack'), ('custom', 'Custom Script')])),
                 ('source_path', models.CharField(default='', max_length=1024, editable=False, blank=True)),
                 ('source_vars', models.TextField(default='', help_text='Inventory source variables in YAML or JSON format.', blank=True)),
@@ -427,7 +427,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Job',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('job_type', models.CharField(default='run', max_length=64, choices=[('run', 'Run'), ('check', 'Check'), ('scan', 'Scan')])),
                 ('playbook', models.CharField(default='', max_length=1024, blank=True)),
                 ('forks', models.PositiveIntegerField(default=0, blank=True)),
@@ -435,7 +435,7 @@ class Migration(migrations.Migration):
                 ('verbosity', models.PositiveIntegerField(default=0, blank=True, choices=[(0, '0 (Normal)'), (1, '1 (Verbose)'), (2, '2 (More Verbose)'), (3, '3 (Debug)'), (4, '4 (Connection Debug)'), (5, '5 (WinRM Debug)')])),
                 ('extra_vars', models.TextField(default='', blank=True)),
                 ('job_tags', models.CharField(default='', max_length=1024, blank=True)),
-                ('force_handlers', models.BooleanField(default=False)),
+                ('force_handlers', models.BooleanField(blank=True, default=False)),
                 ('skip_tags', models.CharField(default='', max_length=1024, blank=True)),
                 ('start_at_task', models.CharField(default='', max_length=1024, blank=True)),
                 ('become_enabled', models.BooleanField(default=False)),
@@ -448,7 +448,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='JobTemplate',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('job_type', models.CharField(default='run', max_length=64, choices=[('run', 'Run'), ('check', 'Check'), ('scan', 'Scan')])),
                 ('playbook', models.CharField(default='', max_length=1024, blank=True)),
                 ('forks', models.PositiveIntegerField(default=0, blank=True)),
@@ -456,14 +456,14 @@ class Migration(migrations.Migration):
                 ('verbosity', models.PositiveIntegerField(default=0, blank=True, choices=[(0, '0 (Normal)'), (1, '1 (Verbose)'), (2, '2 (More Verbose)'), (3, '3 (Debug)'), (4, '4 (Connection Debug)'), (5, '5 (WinRM Debug)')])),
                 ('extra_vars', models.TextField(default='', blank=True)),
                 ('job_tags', models.CharField(default='', max_length=1024, blank=True)),
-                ('force_handlers', models.BooleanField(default=False)),
+                ('force_handlers', models.BooleanField(blank=True, default=False)),
                 ('skip_tags', models.CharField(default='', max_length=1024, blank=True)),
                 ('start_at_task', models.CharField(default='', max_length=1024, blank=True)),
                 ('become_enabled', models.BooleanField(default=False)),
                 ('host_config_key', models.CharField(default='', max_length=1024, blank=True)),
                 ('ask_variables_on_launch', models.BooleanField(default=False)),
                 ('survey_enabled', models.BooleanField(default=False)),
-                ('survey_spec', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('survey_spec', jsonfield.fields.JSONField(default=dict, blank=True)),
             ],
             options={
                 'ordering': ('name',),
@@ -473,7 +473,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Project',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('local_path', models.CharField(help_text='Local path (relative to PROJECTS_ROOT) containing playbooks and related files for this project.', max_length=1024, blank=True)),
                 ('scm_type', models.CharField(default='', max_length=8, verbose_name='SCM Type', blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion')])),
                 ('scm_url', models.CharField(default='', max_length=1024, verbose_name='SCM URL', blank=True)),
@@ -492,7 +492,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='ProjectUpdate',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('local_path', models.CharField(help_text='Local path (relative to PROJECTS_ROOT) containing playbooks and related files for this project.', max_length=1024, blank=True)),
                 ('scm_type', models.CharField(default='', max_length=8, verbose_name='SCM Type', blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion')])),
                 ('scm_url', models.CharField(default='', max_length=1024, verbose_name='SCM URL', blank=True)),
@@ -505,7 +505,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='SystemJob',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('job_type', models.CharField(default='', max_length=32, blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('cleanup_deleted', 'Purge previously deleted items from the database'), ('cleanup_facts', 'Purge and/or reduce the granularity of system tracking data')])),
                 ('extra_vars', models.TextField(default='', blank=True)),
             ],
@@ -517,7 +517,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='SystemJobTemplate',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('job_type', models.CharField(default='', max_length=32, blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('cleanup_deleted', 'Purge previously deleted items from the database'), ('cleanup_facts', 'Purge and/or reduce the granularity of system tracking data')])),
             ],
             bases=('main.unifiedjobtemplate', models.Model),
@@ -550,7 +550,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjobtemplate',
             name='polymorphic_ctype',
-            field=models.ForeignKey(related_name='polymorphic_main.unifiedjobtemplate_set+', editable=False, to='contenttypes.ContentType', null=True),
+            field=models.ForeignKey(related_name='polymorphic_main.unifiedjobtemplate_set+', editable=False, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', null=True),
         ),
         migrations.AddField(
             model_name='unifiedjobtemplate',
@@ -575,7 +575,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjob',
             name='polymorphic_ctype',
-            field=models.ForeignKey(related_name='polymorphic_main.unifiedjob_set+', editable=False, to='contenttypes.ContentType', null=True),
+            field=models.ForeignKey(related_name='polymorphic_main.unifiedjob_set+', editable=False, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', null=True),
         ),
         migrations.AddField(
             model_name='unifiedjob',
@@ -595,7 +595,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='unified_job_template',
-            field=models.ForeignKey(related_name='schedules', to='main.UnifiedJobTemplate'),
+            field=models.ForeignKey(related_name='schedules', on_delete=django.db.models.deletion.CASCADE, to='main.UnifiedJobTemplate'),
         ),
         migrations.AddField(
             model_name='permission',
@@ -610,12 +610,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='joborigin',
             name='unified_job',
-            field=models.OneToOneField(related_name='job_origin', to='main.UnifiedJob'),
+            field=models.OneToOneField(related_name='job_origin', on_delete=django.db.models.deletion.CASCADE, to='main.UnifiedJob'),
         ),
         migrations.AddField(
             model_name='inventory',
             name='organization',
-            field=models.ForeignKey(related_name='inventories', to='main.Organization', help_text='Organization containing this inventory.'),
+            field=models.ForeignKey(related_name='inventories', on_delete=django.db.models.deletion.CASCADE, to='main.Organization', help_text='Organization containing this inventory.'),
         ),
         migrations.AddField(
             model_name='inventory',
@@ -625,7 +625,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='host',
             name='inventory',
-            field=models.ForeignKey(related_name='hosts', to='main.Inventory'),
+            field=models.ForeignKey(related_name='hosts', on_delete=django.db.models.deletion.CASCADE, to='main.Inventory'),
         ),
         migrations.AddField(
             model_name='host',
@@ -650,7 +650,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='group',
             name='inventory',
-            field=models.ForeignKey(related_name='groups', to='main.Inventory'),
+            field=models.ForeignKey(related_name='groups', on_delete=django.db.models.deletion.CASCADE, to='main.Inventory'),
         ),
         migrations.AddField(
             model_name='group',
@@ -680,12 +680,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='credential',
             name='team',
-            field=models.ForeignKey(related_name='credentials', default=None, blank=True, to='main.Team', null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.Team', null=True),
         ),
         migrations.AddField(
             model_name='credential',
             name='user',
-            field=models.ForeignKey(related_name='credentials', default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
         ),
         migrations.AddField(
             model_name='adhoccommandevent',
@@ -774,7 +774,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='projectupdate',
             name='project',
-            field=models.ForeignKey(related_name='project_updates', editable=False, to='main.Project'),
+            field=models.ForeignKey(related_name='project_updates', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.Project'),
         ),
         migrations.AddField(
             model_name='project',
@@ -814,12 +814,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='jobhostsummary',
             name='job',
-            field=models.ForeignKey(related_name='job_host_summaries', editable=False, to='main.Job'),
+            field=models.ForeignKey(related_name='job_host_summaries', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.Job'),
         ),
         migrations.AddField(
             model_name='jobevent',
             name='job',
-            field=models.ForeignKey(related_name='job_events', editable=False, to='main.Job'),
+            field=models.ForeignKey(related_name='job_events', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.Job'),
         ),
         migrations.AddField(
             model_name='job',
@@ -859,7 +859,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventoryupdate',
             name='inventory_source',
-            field=models.ForeignKey(related_name='inventory_updates', editable=False, to='main.InventorySource'),
+            field=models.ForeignKey(related_name='inventory_updates', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.InventorySource'),
         ),
         migrations.AddField(
             model_name='inventoryupdate',
@@ -874,12 +874,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventorysource',
             name='group',
-            field=awx.main.fields.AutoOneToOneField(related_name='inventory_source', null=True, default=None, editable=False, to='main.Group'),
+            field=awx.main.fields.AutoOneToOneField(related_name='inventory_source', on_delete=django.db.models.deletion.SET_NULL, null=True, default=None, editable=False, to='main.Group'),
         ),
         migrations.AddField(
             model_name='inventorysource',
             name='inventory',
-            field=models.ForeignKey(related_name='inventory_sources', default=None, editable=False, to='main.Inventory', null=True),
+            field=models.ForeignKey(related_name='inventory_sources', on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to='main.Inventory', null=True),
         ),
         migrations.AddField(
             model_name='inventorysource',
@@ -916,7 +916,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='adhoccommandevent',
             name='ad_hoc_command',
-            field=models.ForeignKey(related_name='ad_hoc_command_events', editable=False, to='main.AdHocCommand'),
+            field=models.ForeignKey(related_name='ad_hoc_command_events', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.AdHocCommand'),
         ),
         migrations.AddField(
             model_name='adhoccommand',

awx/main/migrations/0002_squashed_v300_release.py

@@ -13,7 +13,6 @@ from django.conf import settings
 from django.utils.timezone import now
 
 import jsonfield.fields
-import jsonbfield.fields
 import taggit.managers
 
 
@@ -144,7 +143,7 @@ class Migration(migrations.Migration):
                 ('category', models.CharField(max_length=128)),
                 ('value', models.TextField(blank=True)),
                 ('value_type', models.CharField(max_length=12, choices=[('string', 'String'), ('int', 'Integer'), ('float', 'Decimal'), ('json', 'JSON'), ('bool', 'Boolean'), ('password', 'Password'), ('list', 'List')])),
-                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True)),
             ],
         ),
         # Notification changes
@@ -185,7 +184,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='notification',
             name='notification_template',
-            field=models.ForeignKey(related_name='notifications', editable=False, to='main.NotificationTemplate'),
+            field=models.ForeignKey(related_name='notifications', editable=False, on_delete=models.CASCADE, to='main.NotificationTemplate'),
         ),
         migrations.AddField(
             model_name='activitystream',
@@ -239,8 +238,8 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('timestamp', models.DateTimeField(default=None, help_text='Date and time of the corresponding fact scan gathering time.', editable=False)),
                 ('module', models.CharField(max_length=128)),
-                ('facts', jsonbfield.fields.JSONField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)),
-                ('host', models.ForeignKey(related_name='facts', to='main.Host', help_text='Host for the facts that the fact scan captured.')),
+                ('facts', awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)),
+                ('host', models.ForeignKey(related_name='facts', to='main.Host', on_delete=models.CASCADE, help_text='Host for the facts that the fact scan captured.')),
             ],
         ),
         migrations.AlterIndexTogether(
@@ -318,7 +317,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='project',
             name='organization',
-            field=models.ForeignKey(related_name='projects', to='main.Organization', blank=True, null=True),
+            field=models.ForeignKey(related_name='projects', to='main.Organization', on_delete=models.CASCADE, blank=True, null=True),
         ),
         migrations.AlterField(
             model_name='team',
@@ -367,7 +366,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='credential',
             name='organization',
-            field=models.ForeignKey(related_name='credentials', default=None, blank=True, to='main.Organization', null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=models.CASCADE, default=None, blank=True, to='main.Organization', null=True),
         ),
 
         #
@@ -382,7 +381,7 @@ class Migration(migrations.Migration):
                 ('members', models.ManyToManyField(related_name='roles', to=settings.AUTH_USER_MODEL)),
                 ('parents', models.ManyToManyField(related_name='children', to='main.Role')),
                 ('implicit_parents', models.TextField(default='[]')),
-                ('content_type', models.ForeignKey(default=None, to='contenttypes.ContentType', null=True)),
+                ('content_type', models.ForeignKey(default=None, to='contenttypes.ContentType', on_delete=models.CASCADE, null=True)),
                 ('object_id', models.PositiveIntegerField(default=None, null=True)),
 
             ],
@@ -398,8 +397,8 @@ class Migration(migrations.Migration):
                 ('role_field', models.TextField()),
                 ('content_type_id', models.PositiveIntegerField()),
                 ('object_id', models.PositiveIntegerField()),
-                ('ancestor', models.ForeignKey(related_name='+', to='main.Role')),
-                ('descendent', models.ForeignKey(related_name='+', to='main.Role')),
+                ('ancestor', models.ForeignKey(on_delete=models.CASCADE, related_name='+', to='main.Role')),
+                ('descendent', models.ForeignKey(on_delete=models.CASCADE, related_name='+', to='main.Role')),
             ],
             options={
                 'db_table': 'main_rbac_role_ancestors',
@@ -569,7 +568,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=512)),
                 ('created_by', models.ForeignKey(related_name="{u'class': 'label', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('modified_by', models.ForeignKey(related_name="{u'class': 'label', u'app_label': 'main'}(class)s_modified+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
-                ('organization', models.ForeignKey(related_name='labels', to='main.Organization', help_text='Organization this label belongs to.')),
+                ('organization', models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.CASCADE, to='main.Organization', help_text='Organization this label belongs to.')),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')),
             ],
             options={
@@ -599,12 +598,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='label',
             name='organization',
-            field=models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.Organization', help_text='Organization this label belongs to.', null=True),
+            field=models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.CASCADE, default=None, blank=True, to='main.Organization', help_text='Organization this label belongs to.', null=True),
         ),
         migrations.AlterField(
             model_name='label',
             name='organization',
-            field=models.ForeignKey(related_name='labels', to='main.Organization', help_text='Organization this label belongs to.'),
+            field=models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.CASCADE, to='main.Organization', help_text='Organization this label belongs to.'),
         ),
         # InventorySource Credential
         migrations.AddField(
@@ -630,12 +629,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='credential',
             name='deprecated_team',
-            field=models.ForeignKey(related_name='deprecated_credentials', default=None, blank=True, to='main.Team', null=True),
+            field=models.ForeignKey(related_name='deprecated_credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.Team', null=True),
         ),
         migrations.AlterField(
             model_name='credential',
             name='deprecated_user',
-            field=models.ForeignKey(related_name='deprecated_credentials', default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
+            field=models.ForeignKey(related_name='deprecated_credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
         ),
         migrations.AlterField(
             model_name='credential',

awx/main/migrations/0003_squashed_v300_v303_updates.py

@@ -116,7 +116,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='team',
             name='organization',
-            field=models.ForeignKey(related_name='teams', to='main.Organization'),
+            field=models.ForeignKey(related_name='teams', on_delete=models.CASCADE, to='main.Organization'),
             preserve_default=False,
         ),
     ] + _squashed.operations(SQUASHED_30, applied=True)

awx/main/migrations/0004_squashed_v310_release.py

@@ -74,7 +74,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='WorkflowJob',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, on_delete=models.CASCADE, primary_key=True, serialize=False, to='main.UnifiedJob')),
                 ('extra_vars', models.TextField(default='', blank=True)),
             ],
             options={
@@ -100,7 +100,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='WorkflowJobTemplate',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=models.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('extra_vars', models.TextField(default='', blank=True)),
                 ('admin_role', awx.main.fields.ImplicitRoleField(related_name='+', parent_role='singleton:system_administrator', to='main.Role', null='True')),
             ],
@@ -116,7 +116,7 @@ class Migration(migrations.Migration):
                 ('failure_nodes', models.ManyToManyField(related_name='workflowjobtemplatenodes_failure', to='main.WorkflowJobTemplateNode', blank=True)),
                 ('success_nodes', models.ManyToManyField(related_name='workflowjobtemplatenodes_success', to='main.WorkflowJobTemplateNode', blank=True)),
                 ('unified_job_template', models.ForeignKey(related_name='workflowjobtemplatenodes', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.UnifiedJobTemplate', null=True)),
-                ('workflow_job_template', models.ForeignKey(related_name='workflow_job_template_nodes', default=None, blank=True, to='main.WorkflowJobTemplate', null=True)),
+                ('workflow_job_template', models.ForeignKey(related_name='workflow_job_template_nodes', on_delete=models.SET_NULL, default=None, blank=True, to='main.WorkflowJobTemplate', null=True)),
             ],
             options={
                 'abstract': False,
@@ -161,7 +161,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobnode',
             name='char_prompts',
-            field=jsonfield.fields.JSONField(default={}, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
@@ -191,7 +191,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='char_prompts',
-            field=jsonfield.fields.JSONField(default={}, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
@@ -211,7 +211,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='workflowjobnode',
             name='workflow_job',
-            field=models.ForeignKey(related_name='workflow_job_nodes', default=None, blank=True, to='main.WorkflowJob', null=True),
+            field=models.ForeignKey(related_name='workflow_job_nodes', on_delete=django.db.models.deletion.CASCADE, default=None, blank=True, to='main.WorkflowJob', null=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplate',
@@ -227,12 +227,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='job',
             name='artifacts',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
             name='ancestor_artifacts',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         # Job timeout settings
         migrations.AddField(
@@ -397,7 +397,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjob',
             name='survey_passwords',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplate',
@@ -407,33 +407,33 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='survey_spec',
-            field=jsonfield.fields.JSONField(default={}, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, blank=True),
         ),
         # JSON field changes
         migrations.AlterField(
             model_name='adhoccommandevent',
             name='event_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='job',
             name='artifacts',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='job',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='jobevent',
             name='event_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='survey_spec',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='notification',
@@ -453,37 +453,37 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='schedule',
             name='extra_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='unifiedjob',
             name='job_env',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjob',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobnode',
             name='ancestor_artifacts',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobnode',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplate',
             name='survey_spec',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplatenode',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         # Job Project Update
         migrations.AddField(

awx/main/migrations/0006_v320_release.py

@@ -55,12 +55,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='inventorysource',
             name='deprecated_group',
-            field=models.OneToOneField(related_name='deprecated_inventory_source', null=True, default=None, to='main.Group'),
+            field=models.OneToOneField(related_name='deprecated_inventory_source', on_delete=models.CASCADE, null=True, default=None, to='main.Group'),
         ),
         migrations.AlterField(
             model_name='inventorysource',
             name='inventory',
-            field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', null=True),
+            field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', on_delete=models.CASCADE, null=True),
         ),
 
         # Smart Inventory
@@ -78,13 +78,13 @@ class Migration(migrations.Migration):
             name='SmartInventoryMembership',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('host', models.ForeignKey(related_name='+', to='main.Host')),
+                ('host', models.ForeignKey(related_name='+', on_delete=models.CASCADE, to='main.Host')),
             ],
         ),
         migrations.AddField(
             model_name='smartinventorymembership',
             name='inventory',
-            field=models.ForeignKey(related_name='+', to='main.Inventory'),
+            field=models.ForeignKey(on_delete=models.CASCADE, related_name='+', to='main.Inventory'),
         ),
         migrations.AddField(
             model_name='host',
@@ -105,19 +105,19 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='inventory',
             name='organization',
-            field=models.ForeignKey(related_name='inventories', on_delete=models.deletion.SET_NULL, to='main.Organization', help_text='Organization containing this inventory.', null=True),
+            field=models.ForeignKey(related_name='inventories', on_delete=models.SET_NULL, to='main.Organization', help_text='Organization containing this inventory.', null=True),
         ),
 
         # Facts
         migrations.AlterField(
             model_name='fact',
             name='facts',
-            field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
+            field=awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
         ),
         migrations.AddField(
             model_name='host',
             name='ansible_facts',
-            field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
+            field=awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
         ),
         migrations.AddField(
             model_name='host',
@@ -148,12 +148,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventorysource',
             name='source_project',
-            field=models.ForeignKey(related_name='scm_inventory_sources', default=None, blank=True, to='main.Project', help_text='Project containing inventory file used as source.', null=True),
+            field=models.ForeignKey(related_name='scm_inventory_sources', on_delete=models.CASCADE, default=None, blank=True, to='main.Project', help_text='Project containing inventory file used as source.', null=True),
         ),
         migrations.AddField(
             model_name='inventoryupdate',
             name='source_project_update',
-            field=models.ForeignKey(related_name='scm_inventory_updates', default=None, blank=True, to='main.ProjectUpdate', help_text='Inventory files from this Project Update were used for the inventory update.', null=True),
+            field=models.ForeignKey(related_name='scm_inventory_updates', on_delete=models.CASCADE, default=None, blank=True, to='main.ProjectUpdate', help_text='Inventory files from this Project Update were used for the inventory update.', null=True),
         ),
         migrations.AddField(
             model_name='project',
@@ -200,7 +200,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='notificationtemplate',
             name='organization',
-            field=models.ForeignKey(related_name='notification_templates', to='main.Organization', null=True),
+            field=models.ForeignKey(related_name='notification_templates', on_delete=models.CASCADE, to='main.Organization', null=True),
         ),
         migrations.AlterUniqueTogether(
             name='notificationtemplate',
@@ -312,7 +312,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventory',
             name='insights_credential',
-            field=models.ForeignKey(related_name='insights_inventories', on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.Credential', help_text='Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.', null=True),
+            field=models.ForeignKey(related_name='insights_inventories', on_delete=models.SET_NULL, default=None, blank=True, to='main.Credential', help_text='Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.', null=True),
         ),
         migrations.AlterField(
             model_name='inventory',
@@ -382,10 +382,10 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=512)),
                 ('kind', models.CharField(max_length=32, choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('insights', 'Insights')])),
                 ('managed_by_tower', models.BooleanField(default=False, editable=False)),
-                ('inputs', awx.main.fields.CredentialTypeInputField(default={}, blank=True, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
-                ('injectors', awx.main.fields.CredentialTypeInjectorField(default={}, blank=True, help_text='Enter injectors using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
-                ('created_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_created+", on_delete=models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
-                ('modified_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_modified+", on_delete=models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('inputs', awx.main.fields.CredentialTypeInputField(default=dict, blank=True, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
+                ('injectors', awx.main.fields.CredentialTypeInjectorField(default=dict, blank=True, help_text='Enter injectors using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
+                ('created_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_created+", on_delete=models.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('modified_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_modified+", on_delete=models.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')),
             ],
             options={
@@ -399,23 +399,23 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='credential',
             name='inputs',
-            field=awx.main.fields.CredentialInputField(default={}, blank=True),
+            field=awx.main.fields.CredentialInputField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='credential',
             name='credential_type',
-            field=models.ForeignKey(related_name='credentials', to='main.CredentialType', null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=models.CASCADE, to='main.CredentialType', null=True),
             preserve_default=False,
         ),
         migrations.AddField(
             model_name='job',
             name='vault_credential',
-            field=models.ForeignKey(related_name='jobs_as_vault_credential+', on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
+            field=models.ForeignKey(related_name='jobs_as_vault_credential+', on_delete=models.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
         ),
         migrations.AddField(
             model_name='jobtemplate',
             name='vault_credential',
-            field=models.ForeignKey(related_name='jobtemplates_as_vault_credential+', on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
+            field=models.ForeignKey(related_name='jobtemplates_as_vault_credential+', on_delete=models.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
         ),
         migrations.AddField(
             model_name='job',
@@ -452,7 +452,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(unique=True, max_length=250)),
                 ('created', models.DateTimeField(auto_now_add=True)),
                 ('modified', models.DateTimeField(auto_now=True)),
-                ('controller', models.ForeignKey(related_name='controlled_groups', default=None, editable=False, to='main.InstanceGroup', help_text='Instance Group to remotely control this group.', null=True)),
+                ('controller', models.ForeignKey(related_name='controlled_groups', on_delete=models.CASCADE, default=None, editable=False, to='main.InstanceGroup', help_text='Instance Group to remotely control this group.', null=True)),
                 ('instances', models.ManyToManyField(help_text='Instances that are members of this InstanceGroup', related_name='rampart_groups', editable=False, to='main.Instance')),
             ],
         ),
@@ -464,7 +464,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjob',
             name='instance_group',
-            field=models.ForeignKey(on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.InstanceGroup', help_text='The Rampart/Instance group the job was run under', null=True),
+            field=models.ForeignKey(on_delete=models.SET_NULL, default=None, blank=True, to='main.InstanceGroup', help_text='The Rampart/Instance group the job was run under', null=True),
         ),
         migrations.AddField(
             model_name='unifiedjobtemplate',

awx/main/migrations/0008_v320_drop_v1_credential_fields.py

@@ -103,12 +103,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='credential',
             name='credential_type',
-            field=models.ForeignKey(related_name='credentials', to='main.CredentialType', null=False, help_text='Specify the type of credential you want to create. Refer to the Ansible Tower documentation for details on each type.')
+            field=models.ForeignKey(related_name='credentials', to='main.CredentialType', on_delete=models.CASCADE, null=False, help_text='Specify the type of credential you want to create. Refer to the Ansible Tower documentation for details on each type.')
         ),
         migrations.AlterField(
             model_name='credential',
             name='inputs',
-            field=awx.main.fields.CredentialInputField(default={}, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.', blank=True),
+            field=awx.main.fields.CredentialInputField(default=dict, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.', blank=True),
         ),
         migrations.RemoveField(
             model_name='job',

awx/main/migrations/0014_v330_saved_launchtime_configs.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='schedule',
@@ -35,7 +35,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
@@ -45,12 +45,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobnode',
             name='extra_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
@@ -60,12 +60,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='extra_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         # Run data migration before removing the old credential field
         migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrations.RunPython.noop),
@@ -83,9 +83,9 @@ class Migration(migrations.Migration):
             name='JobLaunchConfig',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('extra_data', awx.main.fields.JSONField(blank=True, default={})),
-                ('survey_passwords', awx.main.fields.JSONField(blank=True, default={}, editable=False)),
-                ('char_prompts', awx.main.fields.JSONField(blank=True, default={})),
+                ('extra_data', awx.main.fields.JSONField(blank=True, default=dict)),
+                ('survey_passwords', awx.main.fields.JSONField(blank=True, default=dict, editable=False)),
+                ('char_prompts', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('credentials', models.ManyToManyField(related_name='joblaunchconfigs', to='main.Credential')),
                 ('inventory', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='joblaunchconfigs', to='main.Inventory')),
                 ('job', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='launch_config', to='main.UnifiedJob')),
@@ -94,51 +94,51 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='ask_variables_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_credential_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_diff_mode_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_inventory_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_job_type_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_limit_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_skip_tags_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_tags_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_variables_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_verbosity_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
     ]

awx/main/migrations/0018_v330_add_additional_stdout_events.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default={})),
+                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
                 ('counter', models.PositiveIntegerField(default=0, editable=False)),
                 ('stdout', models.TextField(default='', editable=False)),
@@ -40,7 +40,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('event', models.CharField(choices=[('runner_on_failed', 'Host Failed'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_item_on_ok', 'Item OK'), ('runner_item_on_failed', 'Item Failed'), ('runner_item_on_skipped', 'Item Skipped'), ('runner_retry', 'Host Retry'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_include', 'Including File'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete'), ('debug', 'Debug'), ('verbose', 'Verbose'), ('deprecated', 'Deprecated'), ('warning', 'Warning'), ('system_warning', 'System Warning'), ('error', 'Error')], max_length=100)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default={})),
+                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
@@ -65,7 +65,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default={})),
+                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
                 ('counter', models.PositiveIntegerField(default=0, editable=False)),
                 ('stdout', models.TextField(default='', editable=False)),

awx/main/migrations/0053_v340_workflow_inventory.py

@@ -17,7 +17,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjob',
             name='char_prompts',
-            field=awx.main.fields.JSONField(blank=True, default={}),
+            field=awx.main.fields.JSONField(blank=True, default=dict),
         ),
         migrations.AddField(
             model_name='workflowjob',
@@ -27,7 +27,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='ask_inventory_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AddField(
             model_name='workflowjobtemplate',

awx/main/migrations/0067_v350_credential_plugins.py

@@ -34,7 +34,7 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('description', models.TextField(blank=True, default='')),
                 ('input_field_name', models.CharField(max_length=1024)),
-                ('metadata', awx.main.fields.DynamicCredentialInputField(blank=True, default={})),
+                ('metadata', awx.main.fields.DynamicCredentialInputField(blank=True, default=dict)),
                 ('created_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_created+", to=settings.AUTH_USER_MODEL)),
                 ('modified_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_modified+", to=settings.AUTH_USER_MODEL)),
                 ('source_credential', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='target_input_sources', to='main.Credential')),

awx/main/migrations/0081_v360_notify_on_start.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-30 20:35
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+def forwards_split_unified_job_template_any(apps, schema_editor):
+    UnifiedJobTemplate = apps.get_model('main', 'unifiedjobtemplate')
+    for ujt in UnifiedJobTemplate.objects.all():
+        for ujt_notification in ujt.notification_templates_any.all():
+            ujt.notification_templates_success.add(ujt_notification)
+            ujt.notification_templates_error.add(ujt_notification)
+
+def forwards_split_organization_any(apps, schema_editor):
+    Organization = apps.get_model('main', 'organization')
+    for org in Organization.objects.all():
+        for org_notification in org.notification_templates_any.all():
+            org.notification_templates_success.add(org_notification)
+            org.notification_templates_error.add(org_notification)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0080_v360_replace_job_origin'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='organization',
+            name='notification_templates_started',
+            field=models.ManyToManyField(blank=True, related_name='organization_notification_templates_for_started', to='main.NotificationTemplate'),
+        ),
+        migrations.AddField(
+            model_name='unifiedjobtemplate',
+            name='notification_templates_started',
+            field=models.ManyToManyField(blank=True, related_name='unifiedjobtemplate_notification_templates_for_started', to='main.NotificationTemplate'),
+        ),
+        # Separate out "any" notifications into "success" and "error" before the "any" state gets deleted.
+        migrations.RunPython(forwards_split_unified_job_template_any, None),
+        migrations.RunPython(forwards_split_organization_any, None),
+        migrations.RemoveField(
+            model_name='organization',
+            name='notification_templates_any',
+        ),
+        migrations.RemoveField(
+            model_name='unifiedjobtemplate',
+            name='notification_templates_any',
+        ),
+    ]

awx/main/migrations/_rbac.py

@@ -1,17 +1,11 @@
 import logging
 from time import time
 
-from django.utils.encoding import smart_text
-from django.db import transaction
-from django.db.models import Q
-from django.db.utils import IntegrityError
-
-from collections import defaultdict
-from awx.main.utils import getattrd
 from awx.main.models.rbac import Role, batch_role_ancestor_rebuilding
 
 logger = logging.getLogger('rbac_migrations')
 
+
 def create_roles(apps, schema_editor):
     '''
     Implicit role creation happens in our post_save hook for all of our
@@ -41,465 +35,19 @@ def create_roles(apps, schema_editor):
                 obj.save()
 
 
-def migrate_users(apps, schema_editor):
-    User = apps.get_model('auth', "User")
-    Role = apps.get_model('main', "Role")
-    ContentType = apps.get_model('contenttypes', "ContentType")
-    user_content_type = ContentType.objects.get_for_model(User)
-
-    for user in User.objects.iterator():
-        user.save()
-        try:
-            Role.objects.get(content_type=user_content_type, object_id=user.id)
-            logger.info(smart_text(u"found existing role for user: {}".format(user.username)))
-        except Role.DoesNotExist:
-            role = Role.objects.create(
-                role_field='admin_role',
-                content_type = user_content_type,
-                object_id = user.id
-            )
-            role.members.add(user)
-            logger.info(smart_text(u"migrating to new role for user: {}".format(user.username)))
-
-        if user.is_superuser:
-            if Role.objects.filter(singleton_name='system_administrator').exists():
-                sa_role = Role.objects.get(singleton_name='system_administrator')
-            else:
-                sa_role = Role.objects.create(
-                    singleton_name='system_administrator',
-                    role_field='system_administrator'
-                )
-
-            sa_role.members.add(user)
-            logger.warning(smart_text(u"added superuser: {}".format(user.username)))
-
-def migrate_organization(apps, schema_editor):
-    Organization = apps.get_model('main', "Organization")
-    for org in Organization.objects.iterator():
-        for admin in org.deprecated_admins.all():
-            org.admin_role.members.add(admin)
-            logger.info(smart_text(u"added admin: {}, {}".format(org.name, admin.username)))
-        for user in org.deprecated_users.all():
-            org.member_role.members.add(user)
-            logger.info(smart_text(u"added member: {}, {}".format(org.name, user.username)))
-
-def migrate_team(apps, schema_editor):
-    Team = apps.get_model('main', 'Team')
-    for t in Team.objects.iterator():
-        for user in t.deprecated_users.all():
-            t.member_role.members.add(user)
-            logger.info(smart_text(u"team: {}, added user: {}".format(t.name, user.username)))
-
-def attrfunc(attr_path):
-    '''attrfunc returns a function that will
-    attempt to use the attr_path to access the attribute
-    of an instance that is passed in to the returned function.
-
-    Example:
-        get_org = attrfunc('inventory.organization')
-        org = get_org(JobTemplateInstance)
-    '''
-    def attr(inst):
-        return getattrd(inst, attr_path)
-    return attr
-
-def _update_credential_parents(org, cred):
-    cred.organization = org
-    cred.save()
-
-def _discover_credentials(instances, cred, orgfunc):
-    '''_discover_credentials will find shared credentials across
-    organizations. If a shared credential is found, it will duplicate
-    the credential, ensure the proper role permissions are added to the new
-    credential, and update any references from the old to the newly created
-    credential.
-
-    instances is a list of all objects that were matched when filtered
-    with cred.
-
-    orgfunc is a function that when called with an instance from instances
-    will produce an Organization object.
-    '''
-    orgs = defaultdict(list)
-    for inst in instances:
-        try:
-            orgs[orgfunc(inst)].append(inst)
-        except AttributeError:
-            # JobTemplate.inventory can be NULL sometimes, eg when an inventory
-            # has been deleted. This protects against that.
-            pass
-
-    if len(orgs) == 1:
-        try:
-            _update_credential_parents(orgfunc(instances[0]), cred)
-        except AttributeError:
-            # JobTemplate.inventory can be NULL sometimes, eg when an inventory
-            # has been deleted. This protects against that.
-            pass
-    else:
-        for pos, org in enumerate(orgs):
-            if pos == 0:
-                _update_credential_parents(org, cred)
-            else:
-                # Create a new credential
-                cred.pk = None
-                cred.organization = None
-                cred.save()
-
-                cred.admin_role, cred.use_role = None, None
-
-                for i in orgs[org]:
-                    i.credential = cred
-                    i.save()
-
-                _update_credential_parents(org, cred)
-
-def migrate_credential(apps, schema_editor):
-    Credential = apps.get_model('main', "Credential")
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-    Project = apps.get_model('main', 'Project')
-    InventorySource = apps.get_model('main', 'InventorySource')
-
-    for cred in Credential.objects.iterator():
-        results = [x for x in JobTemplate.objects.filter(Q(credential=cred) | Q(cloud_credential=cred), inventory__isnull=False).all()] + \
-                  [x for x in InventorySource.objects.filter(credential=cred).all()]
-        if cred.deprecated_team is not None and results:
-            if len(results) == 1:
-                _update_credential_parents(results[0].inventory.organization, cred)
-            else:
-                _discover_credentials(results, cred, attrfunc('inventory.organization'))
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at organization level".format(cred.name, cred.kind, cred.host)))
-
-        projs = Project.objects.filter(credential=cred).all()
-        if cred.deprecated_team is not None and projs:
-            if len(projs) == 1:
-                _update_credential_parents(projs[0].organization, cred)
-            else:
-                _discover_credentials(projs, cred, attrfunc('organization'))
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at organization level".format(cred.name, cred.kind, cred.host)))
-
-        if cred.deprecated_team is not None:
-            cred.deprecated_team.admin_role.children.add(cred.admin_role)
-            cred.deprecated_team.member_role.children.add(cred.use_role)
-            cred.save()
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at user level".format(cred.name, cred.kind, cred.host)))
-        elif cred.deprecated_user is not None:
-            cred.admin_role.members.add(cred.deprecated_user)
-            cred.save()
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at user level".format(cred.name, cred.kind, cred.host, )))
-        else:
-            logger.warning(smart_text(u"orphaned credential found Credential(name={}, kind={}, host={}), superuser only".format(cred.name, cred.kind, cred.host, )))
-
-
-def migrate_inventory(apps, schema_editor):
-    Inventory = apps.get_model('main', 'Inventory')
-    Permission = apps.get_model('main', 'Permission')
-
-    def role_from_permission(perm):
-        if perm.permission_type == 'admin':
-            return inventory.admin_role
-        elif perm.permission_type == 'read':
-            return inventory.read_role
-        elif perm.permission_type == 'write':
-            return inventory.update_role
-        elif perm.permission_type == 'check' or perm.permission_type == 'run' or perm.permission_type == 'create':
-            # These permission types are handled differntly in RBAC now, nothing to migrate.
-            return False
-        else:
-            return None
-
-    for inventory in Inventory.objects.iterator():
-        for perm in Permission.objects.filter(inventory=inventory):
-            role = None
-            execrole = None
-
-            role = role_from_permission(perm)
-            if role is None:
-                raise Exception(smart_text(u'Unhandled permission type for inventory: {}'.format( perm.permission_type)))
-
-            if perm.run_ad_hoc_commands:
-                execrole = inventory.use_role
-
-            if perm.team:
-                if role:
-                    perm.team.member_role.children.add(role)
-                if execrole:
-                    perm.team.member_role.children.add(execrole)
-                logger.info(smart_text(u'added Team({}) access to Inventory({})'.format(perm.team.name, inventory.name)))
-
-            if perm.user:
-                if role:
-                    role.members.add(perm.user)
-                if execrole:
-                    execrole.members.add(perm.user)
-                logger.info(smart_text(u'added User({}) access to Inventory({})'.format(perm.user.username, inventory.name)))
-
-def migrate_projects(apps, schema_editor):
-    '''
-    I can see projects when:
-     X I am a superuser.
-     X I am an admin in an organization associated with the project.
-     X I am a user in an organization associated with the project.
-     X I am on a team associated with the project.
-     X I have been explicitly granted permission to run/check jobs using the
-       project.
-     X I created the project but it isn't associated with an organization
-    I can change/delete when:
-     X I am a superuser.
-     X I am an admin in an organization associated with the project.
-     X I created the project but it isn't associated with an organization
-    '''
-    Project = apps.get_model('main', 'Project')
-    Permission = apps.get_model('main', 'Permission')
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-
-    # Migrate projects to single organizations, duplicating as necessary
-    for project in Project.objects.iterator():
-        original_project_name = project.name
-        project_orgs = project.deprecated_organizations.distinct().all()
-
-        if len(project_orgs) >= 1:
-            first_org = None
-            for org in project_orgs:
-                if first_org is None:
-                    # For the first org, re-use our existing Project object, so don't do the below duplication effort
-                    first_org = org
-                    if len(project_orgs) > 1:
-                        project.name = smart_text(u'{} - {}'.format(first_org.name, original_project_name))
-                    project.organization = first_org
-                    project.save()
-                else:
-                    new_prj = Project.objects.create(
-                        created                   = project.created,
-                        modified                  = project.modified,
-                        polymorphic_ctype_id      = project.polymorphic_ctype_id,
-                        description               = project.description,
-                        name                      = smart_text(u'{} - {}'.format(org.name, original_project_name)),
-                        old_pk                    = project.old_pk,
-                        created_by_id             = project.created_by_id,
-                        modified_by_id            = project.modified_by_id,
-                        scm_type                  = project.scm_type,
-                        scm_url                   = project.scm_url,
-                        scm_branch                = project.scm_branch,
-                        scm_clean                 = project.scm_clean,
-                        scm_delete_on_update      = project.scm_delete_on_update,
-                        scm_delete_on_next_update = project.scm_delete_on_next_update,
-                        scm_update_on_launch      = project.scm_update_on_launch,
-                        scm_update_cache_timeout  = project.scm_update_cache_timeout,
-                        credential                = project.credential,
-                        organization              = org
-                    )
-                    if project.scm_type == "":
-                        new_prj.local_path = project.local_path
-                        new_prj.save()
-                    for team in project.deprecated_teams.iterator():
-                        new_prj.deprecated_teams.add(team)
-                    logger.warning(smart_text(u'cloning Project({}) onto {} as Project({})'.format(original_project_name, org, new_prj)))
-                    job_templates = JobTemplate.objects.filter(project=project, inventory__organization=org).all()
-                    for jt in job_templates:
-                        jt.project = new_prj
-                        jt.save()
-                    for perm in Permission.objects.filter(project=project):
-                        Permission.objects.create(
-                            created                   = perm.created,
-                            modified                  = perm.modified,
-                            created_by                = perm.created_by,
-                            modified_by               = perm.modified_by,
-                            description               = perm.description,
-                            name                      = perm.name,
-                            user                      = perm.user,
-                            team                      = perm.team,
-                            project                   = new_prj,
-                            inventory                 = perm.inventory,
-                            permission_type           = perm.permission_type,
-                            run_ad_hoc_commands       = perm.run_ad_hoc_commands,
-                        )
-
-    # Migrate permissions
-    for project in Project.objects.iterator():
-        if project.organization is None and project.created_by is not None:
-            project.admin_role.members.add(project.created_by)
-            logger.warn(smart_text(u'adding Project({}) admin: {}'.format(project.name, project.created_by.username)))
-
-        for team in project.deprecated_teams.all():
-            team.member_role.children.add(project.read_role)
-            logger.info(smart_text(u'adding Team({}) access for Project({})'.format(team.name, project.name)))
-
-        for perm in Permission.objects.filter(project=project):
-            if perm.permission_type == 'create':
-                role = project.use_role
-            else:
-                role = project.read_role
-
-            if perm.team:
-                perm.team.member_role.children.add(role)
-                logger.info(smart_text(u'adding Team({}) access for Project({})'.format(perm.team.name, project.name)))
-
-            if perm.user:
-                role.members.add(perm.user)
-                logger.info(smart_text(u'adding User({}) access for Project({})'.format(perm.user.username, project.name)))
-
-        if project.organization is not None:
-            for user in project.organization.deprecated_users.all():
-                if not (project.use_role.members.filter(pk=user.id).exists() or project.admin_role.members.filter(pk=user.id).exists()):
-                    project.read_role.members.add(user)
-                    logger.info(smart_text(u'adding Organization({}) member access to Project({})'.format(project.organization.name, project.name)))
-
-
-
-def migrate_job_templates(apps, schema_editor):
-    '''
-    NOTE: This must be run after orgs, inventory, projects, credential, and
-    users have been migrated
-    '''
-
-
-    '''
-    I can see job templates when:
-     X I am a superuser.
-     - I can read the inventory, project and credential (which means I am an
-       org admin or member of a team with access to all of the above).
-     - I have permission explicitly granted to check/deploy with the inventory
-       and project.
-
-
-    #This does not mean I would be able to launch a job from the template or
-    #edit the template.
-     - access.py can_read for JobTemplate enforces that you can only
-       see it if you can launch it, so the above imply launch too
-    '''
-
-
-    '''
-    Tower administrators, organization administrators, and project
-    administrators, within a project under their purview, may create and modify
-    new job templates for that project.
-
-    When editing a job template, they may select among the inventory groups and
-    credentials in the organization for which they have usage permissions, or
-    they may leave either blank to be selected at runtime.
-
-    Additionally, they may specify one or more users/teams that have execution
-    permission for that job template, among the users/teams that are a member
-    of that project.
-
-    That execution permission is valid irrespective of any explicit permissions
-    the user has or has not been granted to the inventory group or credential
-    specified in the job template.
-
-    '''
-
-    User = apps.get_model('auth', 'User')
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-    Team = apps.get_model('main', 'Team')
-    Permission = apps.get_model('main', 'Permission')
-    Credential = apps.get_model('main', 'Credential')
-
-    jt_queryset = JobTemplate.objects.select_related('inventory', 'project', 'inventory__organization', 'execute_role')
-
-    for jt in jt_queryset.iterator():
-        if jt.inventory is None:
-            # If inventory is None, then only system admins and org admins can
-            # do anything with the JT in 2.4
-            continue
-
-        jt_permission_qs = Permission.objects.filter(
-            inventory=jt.inventory,
-            project=jt.project,
-        )
-
-        inventory_permission_qs = Permission.objects.filter(
-            inventory=jt.inventory,
-            project__isnull=True,
-        )
-
-        team_create_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['create'])
-            .values_list('team__id', flat=True)
-        )
-        team_run_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['check', 'run'] if jt.job_type == 'check' else ['run'])
-            .values_list('team__id', flat=True)
-        )
-        user_create_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['create'])
-            .values_list('user__id', flat=True)
-        )
-        user_run_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['check', 'run'] if jt.job_type == 'check' else ['run'])
-            .values_list('user__id', flat=True)
-        )
-
-        team_inv_permissions = defaultdict(set)
-        user_inv_permissions = defaultdict(set)
-
-        for user_id, team_id, inventory_id in inventory_permission_qs.values_list('user_id', 'team_id', 'inventory_id'):
-            if user_id:
-                user_inv_permissions[user_id].add(inventory_id)
-            if team_id:
-                team_inv_permissions[team_id].add(inventory_id)
-
-
-        for team in Team.objects.filter(id__in=team_create_permissions).iterator():
-            if jt.inventory.id in team_inv_permissions[team.id] and \
-                ((not jt.credential and not jt.cloud_credential) or
-                    Credential.objects.filter(deprecated_team=team, jobtemplates=jt).exists()):
-                team.member_role.children.add(jt.admin_role)
-                logger.info(smart_text(u'transfering admin access on JobTemplate({}) to Team({})'.format(jt.name, team.name)))
-        for team in Team.objects.filter(id__in=team_run_permissions).iterator():
-            if jt.inventory.id in team_inv_permissions[team.id] and \
-               ((not jt.credential and not jt.cloud_credential) or
-                    Credential.objects.filter(deprecated_team=team, jobtemplates=jt).exists()):
-                team.member_role.children.add(jt.execute_role)
-                logger.info(smart_text(u'transfering execute access on JobTemplate({}) to Team({})'.format(jt.name, team.name)))
-
-        for user in User.objects.filter(id__in=user_create_permissions).iterator():
-            cred = jt.credential or jt.cloud_credential
-            if (jt.inventory.id in user_inv_permissions[user.id] or
-                    any([jt.inventory.id in team_inv_permissions[team.id] for team in user.deprecated_teams.all()])) and \
-                    (not cred or cred.deprecated_user == user or
-                        (cred.deprecated_team and cred.deprecated_team.deprecated_users.filter(pk=user.id).exists())):
-                jt.admin_role.members.add(user)
-                logger.info(smart_text(u'transfering admin access on JobTemplate({}) to User({})'.format(jt.name, user.username)))
-        for user in User.objects.filter(id__in=user_run_permissions).iterator():
-            cred = jt.credential or jt.cloud_credential
-
-            if (jt.inventory.id in user_inv_permissions[user.id] or
-                    any([jt.inventory.id in team_inv_permissions[team.id] for team in user.deprecated_teams.all()])) and \
-                    (not cred or cred.deprecated_user == user or
-                        (cred.deprecated_team and cred.deprecated_team.deprecated_users.filter(pk=user.id).exists())):
-                jt.execute_role.members.add(user)
-                logger.info(smart_text(u'transfering execute access on JobTemplate({}) to User({})'.format(jt.name, user.username)))
-
-
-
 def rebuild_role_hierarchy(apps, schema_editor):
-        logger.info('Computing role roots..')
-        start = time()
-        roots = Role.objects \
-                    .all() \
-                    .values_list('id', flat=True)
-        stop = time()
-        logger.info('Found %d roots in %f seconds, rebuilding ancestry map' % (len(roots), stop - start))
-        start = time()
-        Role.rebuild_role_ancestor_list(roots, [])
-        stop = time()
-        logger.info('Rebuild completed in %f seconds' % (stop - start))
-        logger.info('Done.')
-
-
-def infer_credential_org_from_team(apps, schema_editor):
-    Credential = apps.get_model('main', "Credential")
-    for cred in Credential.objects.exclude(deprecated_team__isnull=True):
-        try:
-            with transaction.atomic():
-                _update_credential_parents(cred.deprecated_team.organization, cred)
-        except IntegrityError:
-            logger.info("Organization<{}> credential for old Team<{}> credential already created".format(cred.deprecated_team.organization.pk, cred.pk))
+    logger.info('Computing role roots..')
+    start = time()
+    roots = Role.objects \
+                .all() \
+                .values_list('id', flat=True)
+    stop = time()
+    logger.info('Found %d roots in %f seconds, rebuilding ancestry map' % (len(roots), stop - start))
+    start = time()
+    Role.rebuild_role_ancestor_list(roots, [])
+    stop = time()
+    logger.info('Rebuild completed in %f seconds' % (stop - start))
+    logger.info('Done.')
 
 
 def delete_all_user_roles(apps, schema_editor):

awx/main/migrations/_squashed_30.py

@@ -30,7 +30,7 @@ SQUASHED_30 = {
         migrations.AddField(
             model_name='job',
             name='survey_passwords',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
     ],
     '0031_v302_migrate_survey_passwords': [

awx/main/models/__init__.py

@@ -62,24 +62,6 @@ from awx.main.models.oauth import ( # noqa
 from oauth2_provider.models import Grant, RefreshToken # noqa -- needed django-oauth-toolkit model migrations
 
 
-
-# Monkeypatch Django serializer to ignore django-taggit fields (which break
-# the dumpdata command; see https://github.com/alex/django-taggit/issues/155).
-from django.core.serializers.python import Serializer as _PythonSerializer
-_original_handle_m2m_field = _PythonSerializer.handle_m2m_field
-
-
-def _new_handle_m2m_field(self, obj, field):
-    try:
-        field.rel.through._meta
-    except AttributeError:
-        return
-    return _original_handle_m2m_field(self, obj, field)
-
-
-_PythonSerializer.handle_m2m_field = _new_handle_m2m_field
-
-
 # Add custom methods to User model for permissions checks.
 from django.contrib.auth.models import User  # noqa
 from awx.main.access import (  # noqa
@@ -158,7 +140,7 @@ User.add_to_class('is_system_auditor', user_is_system_auditor)
 
 
 def user_is_in_enterprise_category(user, category):
-    ret = (category,) in user.enterprise_auth.all().values_list('provider') and not user.has_usable_password()
+    ret = (category,) in user.enterprise_auth.values_list('provider') and not user.has_usable_password()
     # NOTE: this if-else block ensures existing enterprise users are still able to
     # log in. Remove it in a future release
     if category == 'radius':

awx/main/models/ad_hoc_commands.py

@@ -163,18 +163,18 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
             all_orgs.add(h.inventory.organization)
         active_templates = dict(error=set(),
                                 success=set(),
-                                any=set())
+                                started=set())
         base_notification_templates = NotificationTemplate.objects
         for org in all_orgs:
             for templ in base_notification_templates.filter(organization_notification_templates_for_errors=org):
                 active_templates['error'].add(templ)
             for templ in base_notification_templates.filter(organization_notification_templates_for_success=org):
                 active_templates['success'].add(templ)
-            for templ in base_notification_templates.filter(organization_notification_templates_for_any=org):
-                active_templates['any'].add(templ)
+            for templ in base_notification_templates.filter(organization_notification_templates_for_started=org):
+                active_templates['started'].add(templ)
         active_templates['error'] = list(active_templates['error'])
-        active_templates['any'] = list(active_templates['any'])
         active_templates['success'] = list(active_templates['success'])
+        active_templates['started'] = list(active_templates['started'])
         return active_templates
 
     def get_passwords_needed_to_start(self):

awx/main/models/base.py

@@ -386,14 +386,13 @@ class NotificationFieldsModel(BaseModel):
         related_name='%(class)s_notification_templates_for_success'
     )
 
-    notification_templates_any = models.ManyToManyField(
+    notification_templates_started = models.ManyToManyField(
         "NotificationTemplate",
         blank=True,
-        related_name='%(class)s_notification_templates_for_any'
+        related_name='%(class)s_notification_templates_for_started'
     )
 
 
-
 def prevent_search(relation):
     """
     Used to mark a model field or relation as "restricted from filtering"

awx/main/models/credential/__init__.py

@@ -105,7 +105,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     )
     inputs = CredentialInputField(
         blank=True,
-        default={},
+        default=dict,
         help_text=_('Enter inputs using either JSON or YAML syntax. Use the '
                     'radio button to toggle between the two. Refer to the '
                     'Ansible Tower documentation for example syntax.')
@@ -343,14 +343,14 @@ class CredentialType(CommonModelNameNotUnique):
     )
     inputs = CredentialTypeInputField(
         blank=True,
-        default={},
+        default=dict,
         help_text=_('Enter inputs using either JSON or YAML syntax. Use the '
                     'radio button to toggle between the two. Refer to the '
                     'Ansible Tower documentation for example syntax.')
     )
     injectors = CredentialTypeInjectorField(
         blank=True,
-        default={},
+        default=dict,
         help_text=_('Enter injectors using either JSON or YAML syntax. Use the '
                     'radio button to toggle between the two. Refer to the '
                     'Ansible Tower documentation for example syntax.')
@@ -1117,7 +1117,7 @@ class CredentialInputSource(PrimordialModel):
     )
     metadata = DynamicCredentialInputField(
         blank=True,
-        default={}
+        default=dict
     )
 
     def clean_target_credential(self):

awx/main/models/credential/injectors.py

@@ -39,6 +39,14 @@ def gce(cred, env, private_data_dir):
     f.close()
     os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
     env['GCE_CREDENTIALS_FILE_PATH'] = path
+    env['GCP_SERVICE_ACCOUNT_FILE'] = path
+
+    # Handle env variables for new module types.
+    # This includes gcp_compute inventory plugin and
+    # all new gcp_* modules.
+    env['GCP_AUTH_KIND'] = 'serviceaccount'
+    env['GCP_PROJECT'] = project
+    env['GCP_ENV_TYPE'] = 'tower'
     return path
 
 

awx/main/models/events.py

@@ -149,7 +149,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
     )
     event_data = JSONField(
         blank=True,
-        default={},
+        default=dict,
     )
     failed = models.BooleanField(
         default=False,
@@ -567,7 +567,7 @@ class BaseCommandEvent(CreatedModifiedModel):
 
     event_data = JSONField(
         blank=True,
-        default={},
+        default=dict,
     )
     uuid = models.CharField(
         max_length=1024,

awx/main/models/inventory.py

@@ -650,7 +650,7 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
     )
     ansible_facts = JSONBField(
         blank=True,
-        default={},
+        default=dict,
         help_text=_('Arbitrary JSON structure of most recent ansible_facts, per-host.'),
     )
     ansible_facts_modified = models.DateTimeField(
@@ -1619,20 +1619,20 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
         base_notification_templates = NotificationTemplate.objects
         error_notification_templates = list(base_notification_templates
                                             .filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
+        started_notification_templates = list(base_notification_templates
+                                              .filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
         success_notification_templates = list(base_notification_templates
                                               .filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
-        any_notification_templates = list(base_notification_templates
-                                          .filter(unifiedjobtemplate_notification_templates_for_any__in=[self]))
         if self.inventory.organization is not None:
             error_notification_templates = set(error_notification_templates + list(base_notification_templates
                                                .filter(organization_notification_templates_for_errors=self.inventory.organization)))
+            started_notification_templates = set(started_notification_templates + list(base_notification_templates
+                                                 .filter(organization_notification_templates_for_started=self.inventory.organization)))
             success_notification_templates = set(success_notification_templates + list(base_notification_templates
                                                  .filter(organization_notification_templates_for_success=self.inventory.organization)))
-            any_notification_templates = set(any_notification_templates + list(base_notification_templates
-                                             .filter(organization_notification_templates_for_any=self.inventory.organization)))
         return dict(error=list(error_notification_templates),
-                    success=list(success_notification_templates),
-                    any=list(any_notification_templates))
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     def clean_source(self):  # TODO: remove in 3.3
         source = self.source
@@ -1991,6 +1991,8 @@ class azure_rm(PluginFileInjector):
 
         source_vars = inventory_update.source_vars_dict
 
+        ret['fail_on_template_errors'] = False
+
         group_by_hostvar = {
             'location': {'prefix': '', 'separator': '', 'key': 'location'},
             'tag': {'prefix': '', 'separator': '', 'key': 'tags.keys() | list if tags else []'},

awx/main/models/jobs.py

@@ -435,19 +435,21 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         base_notification_templates = NotificationTemplate.objects
         error_notification_templates = list(base_notification_templates.filter(
             unifiedjobtemplate_notification_templates_for_errors__in=[self, self.project]))
+        started_notification_templates = list(base_notification_templates.filter(
+            unifiedjobtemplate_notification_templates_for_started__in=[self, self.project]))
         success_notification_templates = list(base_notification_templates.filter(
             unifiedjobtemplate_notification_templates_for_success__in=[self, self.project]))
-        any_notification_templates = list(base_notification_templates.filter(
-            unifiedjobtemplate_notification_templates_for_any__in=[self, self.project]))
         # Get Organization NotificationTemplates
         if self.project is not None and self.project.organization is not None:
             error_notification_templates = set(error_notification_templates + list(base_notification_templates.filter(
                 organization_notification_templates_for_errors=self.project.organization)))
+            started_notification_templates = set(started_notification_templates + list(base_notification_templates.filter(
+                organization_notification_templates_for_started=self.project.organization)))
             success_notification_templates = set(success_notification_templates + list(base_notification_templates.filter(
                 organization_notification_templates_for_success=self.project.organization)))
-            any_notification_templates = set(any_notification_templates + list(base_notification_templates.filter(
-                organization_notification_templates_for_any=self.project.organization)))
-        return dict(error=list(error_notification_templates), success=list(success_notification_templates), any=list(any_notification_templates))
+        return dict(error=list(error_notification_templates),
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     '''
     RelatedJobsMixin
@@ -483,7 +485,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
     )
     artifacts = JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     )
     scm_revision = models.CharField(
@@ -845,7 +847,7 @@ class LaunchTimeConfigBase(BaseModel):
     # This is a solution to the nullable CharField problem, specific to prompting
     char_prompts = JSONField(
         blank=True,
-        default={}
+        default=dict
     )
 
     def prompts_dict(self, display=False):
@@ -925,11 +927,11 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
     # Special case prompting fields, even more special than the other ones
     extra_data = JSONField(
         blank=True,
-        default={}
+        default=dict
     )
     survey_passwords = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     ))
     # Credentials needed for non-unified job / unified JT models
@@ -1133,13 +1135,13 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):
         base_notification_templates = NotificationTemplate.objects.all()
         error_notification_templates = list(base_notification_templates
                                             .filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
+        started_notification_templates = list(base_notification_templates
+                                              .filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
         success_notification_templates = list(base_notification_templates
                                               .filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
-        any_notification_templates = list(base_notification_templates
-                                          .filter(unifiedjobtemplate_notification_templates_for_any__in=[self]))
         return dict(error=list(error_notification_templates),
-                    success=list(success_notification_templates),
-                    any=list(any_notification_templates))
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     def _accept_or_ignore_job_kwargs(self, _exclude_errors=None, **kwargs):
         extra_data = kwargs.pop('extra_vars', {})

awx/main/models/mixins.py

@@ -100,7 +100,7 @@ class SurveyJobTemplateMixin(models.Model):
     )
     survey_spec = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
     ))
     ask_variables_on_launch = AskForField(
         blank=True,
@@ -360,7 +360,7 @@ class SurveyJobMixin(models.Model):
 
     survey_passwords = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     ))
 

awx/main/models/notifications.py

@@ -7,6 +7,7 @@ import logging
 from django.db import models
 from django.conf import settings
 from django.core.mail.message import EmailMessage
+from django.db import connection
 from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import smart_str, force_text
 
@@ -129,7 +130,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
             if field not in notification_configuration:
                 if 'default' in params:
                     notification_configuration[field] = params['default']
-        backend_obj = self.notification_class(**notification_configuration) 
+        backend_obj = self.notification_class(**notification_configuration)
         notification_obj = EmailMessage(subject, backend_obj.format_body(body), sender, recipients)
         with set_environ(**settings.AWX_TASK_ENV):
             return backend_obj.send_messages([notification_obj])
@@ -221,10 +222,13 @@ class JobNotificationMixin(object):
     def build_notification_failed_message(self):
         return self._build_notification_message('failed')
 
+    def build_notification_running_message(self):
+        return self._build_notification_message('running')
+
     def send_notification_templates(self, status_str):
         from awx.main.tasks import send_notifications  # avoid circular import
-        if status_str not in ['succeeded', 'failed']:
-            raise ValueError(_("status_str must be either succeeded or failed"))
+        if status_str not in ['succeeded', 'failed', 'running']:
+            raise ValueError(_("status_str must be either running, succeeded or failed"))
         try:
             notification_templates = self.get_notification_templates()
         except Exception:
@@ -233,14 +237,19 @@ class JobNotificationMixin(object):
         if notification_templates:
             if status_str == 'succeeded':
                 notification_template_type = 'success'
+            elif status_str == 'running':
+                notification_template_type = 'started'
             else:
                 notification_template_type = 'error'
-            all_notification_templates = set(notification_templates.get(notification_template_type, []) + notification_templates.get('any', []))
+            all_notification_templates = set(notification_templates.get(notification_template_type, []))
             if len(all_notification_templates):
                 try:
                     (notification_subject, notification_body) = getattr(self, 'build_notification_%s_message' % status_str)()
                 except AttributeError:
                     raise NotImplementedError("build_notification_%s_message() does not exist" % status_str)
-                send_notifications.delay([n.generate_notification(notification_subject, notification_body).id
-                                          for n in all_notification_templates],
-                                         job_id=self.id)
+
+                def send_it():
+                    send_notifications.delay([n.generate_notification(notification_subject, notification_body).id
+                                              for n in all_notification_templates],
+                                             job_id=self.id)
+                connection.on_commit(send_it)

awx/main/models/projects.py

@@ -411,24 +411,24 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         base_notification_templates = NotificationTemplate.objects
         error_notification_templates = list(base_notification_templates
                                             .filter(unifiedjobtemplate_notification_templates_for_errors=self))
+        started_notification_templates = list(base_notification_templates
+                                              .filter(unifiedjobtemplate_notification_templates_for_started=self))
         success_notification_templates = list(base_notification_templates
                                               .filter(unifiedjobtemplate_notification_templates_for_success=self))
-        any_notification_templates = list(base_notification_templates
-                                          .filter(unifiedjobtemplate_notification_templates_for_any=self))
         # Get Organization NotificationTemplates
         if self.organization is not None:
             error_notification_templates = set(error_notification_templates +
                                                list(base_notification_templates
                                                     .filter(organization_notification_templates_for_errors=self.organization)))
+            started_notification_templates = set(started_notification_templates +
+                                                 list(base_notification_templates
+                                                      .filter(organization_notification_templates_for_started=self.organization)))
             success_notification_templates = set(success_notification_templates +
                                                  list(base_notification_templates
                                                       .filter(organization_notification_templates_for_success=self.organization)))
-            any_notification_templates = set(any_notification_templates +
-                                             list(base_notification_templates
-                                                  .filter(organization_notification_templates_for_any=self.organization)))
         return dict(error=list(error_notification_templates),
-                    success=list(success_notification_templates),
-                    any=list(any_notification_templates))
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     def get_absolute_url(self, request=None):
         return reverse('api:project_detail', kwargs={'pk': self.pk}, request=request)
@@ -567,5 +567,3 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
         if not selected_groups:
             return self.global_instance_groups
         return selected_groups
-
-

awx/main/models/unified_jobs.py

@@ -641,7 +641,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
     )
     job_env = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     ))
     job_explanation = models.TextField(
@@ -1397,6 +1397,13 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
                 r['{}_user_email'.format(name)] = created_by.email
                 r['{}_user_first_name'.format(name)] = created_by.first_name
                 r['{}_user_last_name'.format(name)] = created_by.last_name
+
+        inventory = getattr_dne(self, 'inventory')
+        if inventory:
+            for name in ('awx', 'tower'):
+                r['{}_inventory_id'.format(name)] = inventory.pk
+                r['{}_inventory_name'.format(name)] = inventory.name
+
         return r
 
     def get_queue_name(self):

awx/main/models/workflow.py

@@ -180,7 +180,7 @@ class WorkflowJobNode(WorkflowNodeBase):
     )
     ancestor_artifacts = JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     )
     do_not_run = models.BooleanField(
@@ -419,13 +419,13 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
         base_notification_templates = NotificationTemplate.objects.all()
         error_notification_templates = list(base_notification_templates
                                             .filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
+        started_notification_templates = list(base_notification_templates
+                                              .filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
         success_notification_templates = list(base_notification_templates
                                               .filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
-        any_notification_templates = list(base_notification_templates
-                                          .filter(unifiedjobtemplate_notification_templates_for_any__in=[self]))
         return dict(error=list(error_notification_templates),
-                    success=list(success_notification_templates),
-                    any=list(any_notification_templates))
+                    started=list(started_notification_templates),
+                    success=list(success_notification_templates))
 
     def create_unified_job(self, **kwargs):
         workflow_job = super(WorkflowJobTemplate, self).create_unified_job(**kwargs)

awx/main/scheduler/task_manager.py

@@ -193,6 +193,8 @@ class TaskManager():
                 status_changed = True
             if status_changed:
                 workflow_job.websocket_emit_status(workflow_job.status)
+                # Operations whose queries rely on modifications made during the atomic scheduling session
+                workflow_job.send_notification_templates('succeeded' if workflow_job.status == 'successful' else 'failed')
                 if workflow_job.spawned_by_workflow:
                     schedule_task_manager()
         return result
@@ -233,6 +235,7 @@ class TaskManager():
         else:
             if type(task) is WorkflowJob:
                 task.status = 'running'
+                task.send_notification_templates('running')
                 logger.debug('Transitioning %s to running status.', task.log_format)
                 schedule_task_manager()
             elif not task.supports_isolation() and rampart_group.controller_id:
@@ -581,10 +584,5 @@ class TaskManager():
                     logger.debug("Not running scheduler, another task holds lock")
                     return
                 logger.debug("Starting Scheduler")
-
                 with task_manager_bulk_reschedule():
-                    finished_wfjs = self._schedule()
-
-                # Operations whose queries rely on modifications made during the atomic scheduling session
-                for wfj in WorkflowJob.objects.filter(id__in=finished_wfjs):
-                    wfj.send_notification_templates('succeeded' if wfj.status == 'successful' else 'failed')
+                    self._schedule()

awx/main/tasks.py

@@ -602,25 +602,26 @@ def update_inventory_computed_fields(inventory_id, should_update_hosts=True):
 
 
 def update_smart_memberships_for_inventory(smart_inventory):
-    current = set(SmartInventoryMembership.objects.filter(inventory=smart_inventory).values_list('host_id', flat=True))
-    new = set(smart_inventory.hosts.values_list('id', flat=True))
-    additions = new - current
-    removals = current - new
-    if additions or removals:
-        with transaction.atomic():
-            if removals:
-                SmartInventoryMembership.objects.filter(inventory=smart_inventory, host_id__in=removals).delete()
-            if additions:
-                add_for_inventory = [
-                    SmartInventoryMembership(inventory_id=smart_inventory.id, host_id=host_id)
-                    for host_id in additions
-                ]
-                SmartInventoryMembership.objects.bulk_create(add_for_inventory)
-        logger.debug('Smart host membership cached for {}, {} additions, {} removals, {} total count.'.format(
-            smart_inventory.pk, len(additions), len(removals), len(new)
-        ))
-        return True  # changed
-    return False
+    with advisory_lock('update_smart_memberships_for_inventory-{}'.format(smart_inventory.id)):
+        current = set(SmartInventoryMembership.objects.filter(inventory=smart_inventory).values_list('host_id', flat=True))
+        new = set(smart_inventory.hosts.values_list('id', flat=True))
+        additions = new - current
+        removals = current - new
+        if additions or removals:
+            with transaction.atomic():
+                if removals:
+                    SmartInventoryMembership.objects.filter(inventory=smart_inventory, host_id__in=removals).delete()
+                if additions:
+                    add_for_inventory = [
+                        SmartInventoryMembership(inventory_id=smart_inventory.id, host_id=host_id)
+                        for host_id in additions
+                    ]
+                    SmartInventoryMembership.objects.bulk_create(add_for_inventory)
+            logger.debug('Smart host membership cached for {}, {} additions, {} removals, {} total count.'.format(
+                smart_inventory.pk, len(additions), len(removals), len(new)
+            ))
+            return True  # changed
+        return False
 
 
 @task()
@@ -827,8 +828,10 @@ class BaseTask(object):
                     os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
                 private_data_files['credentials'][credential] = path
             for credential, data in private_data.get('certificates', {}).items():
-                name = 'credential_%d-cert.pub' % credential.pk
-                path = os.path.join(private_data_dir, name)
+                artifact_dir = os.path.join(private_data_dir, 'artifacts', str(self.instance.id))
+                if not os.path.exists(artifact_dir):
+                    os.makedirs(artifact_dir, mode=0o700)
+                path = os.path.join(artifact_dir, 'ssh_key_data-cert.pub')
                 with open(path, 'w') as f:
                     f.write(data)
                     f.close()
@@ -856,10 +859,16 @@ class BaseTask(object):
         '''
         process_isolation_params = dict()
         if self.should_use_proot(instance):
+            show_paths = self.proot_show_paths + [private_data_dir, cwd] + \
+                settings.AWX_PROOT_SHOW_PATHS
+
+            # Help the user out by including the collections path inside the bubblewrap environment
+            if getattr(settings, 'AWX_ANSIBLE_COLLECTIONS_PATHS', []):
+                show_paths.extend(settings.AWX_ANSIBLE_COLLECTIONS_PATHS)
             process_isolation_params = {
                 'process_isolation': True,
                 'process_isolation_path': settings.AWX_PROOT_BASE_PATH,
-                'process_isolation_show_paths': self.proot_show_paths + [private_data_dir, cwd] + settings.AWX_PROOT_SHOW_PATHS,
+                'process_isolation_show_paths': show_paths,
                 'process_isolation_hide_paths': [
                     settings.AWX_PROOT_BASE_PATH,
                     '/etc/tower',
@@ -933,6 +942,11 @@ class BaseTask(object):
         if self.should_use_proot(instance):
             env['PROOT_TMP_DIR'] = settings.AWX_PROOT_BASE_PATH
         env['AWX_PRIVATE_DATA_DIR'] = private_data_dir
+
+        if 'ANSIBLE_COLLECTIONS_PATHS' in env:
+            env['ANSIBLE_COLLECTIONS_PATHS'] += os.pathsep + os.pathsep.join(settings.AWX_ANSIBLE_COLLECTIONS_PATHS)
+        else:
+            env['ANSIBLE_COLLECTIONS_PATHS'] = os.pathsep.join(settings.AWX_ANSIBLE_COLLECTIONS_PATHS)
         return env
 
     def should_use_proot(self, instance):
@@ -1131,6 +1145,7 @@ class BaseTask(object):
 
         try:
             isolated = self.instance.is_isolated()
+            self.instance.send_notification_templates("running")
             self.pre_run_hook(self.instance)
             if self.instance.cancel_flag:
                 self.instance = self.update_model(self.instance.pk, status='canceled')

awx/main/tests/data/inventory/plugins/azure_rm/env.json

@@ -1,9 +1,9 @@
 {
-    "AZURE_SUBSCRIPTION_ID": "fooo",
-    "AZURE_CLIENT_ID": "fooo",
-    "AZURE_TENANT": "fooo",
-    "AZURE_SECRET": "fooo",
-    "AZURE_CLOUD_ENVIRONMENT": "fooo",
     "ANSIBLE_JINJA2_NATIVE": "True",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
+    "AZURE_CLIENT_ID": "fooo",
+    "AZURE_CLOUD_ENVIRONMENT": "fooo",
+    "AZURE_SECRET": "fooo",
+    "AZURE_SUBSCRIPTION_ID": "fooo",
+    "AZURE_TENANT": "fooo"
 }

awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml

@@ -4,6 +4,7 @@ default_host_filters: []
 exclude_host_filters:
 - resource_group not in ['foo_resources', 'bar_resources']
 - location not in ['southcentralus', 'westus']
+fail_on_template_errors: false
 hostvar_expressions:
   ansible_host: private_ipv4_addresses[0]
   computer_name: name

awx/main/tests/data/inventory/plugins/ec2/env.json

@@ -1,7 +1,7 @@
 {
+    "ANSIBLE_JINJA2_NATIVE": "True",
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "AWS_ACCESS_KEY_ID": "fooo",
     "AWS_SECRET_ACCESS_KEY": "fooo",
-    "AWS_SECURITY_TOKEN": "fooo",
-    "ANSIBLE_JINJA2_NATIVE": "True",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "AWS_SECURITY_TOKEN": "fooo"
 }

awx/main/tests/data/inventory/plugins/gce/env.json

@@ -1,4 +1,8 @@
 {
     "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
-    "GCE_CREDENTIALS_FILE_PATH": "{{ file_reference }}"
+    "GCE_CREDENTIALS_FILE_PATH": "{{ file_reference }}",
+    "GCP_AUTH_KIND": "serviceaccount",
+    "GCP_ENV_TYPE": "tower",
+    "GCP_PROJECT": "fooo",
+    "GCP_SERVICE_ACCOUNT_FILE": "{{ file_reference }}"
 }

awx/main/tests/data/inventory/plugins/satellite6/env.json

@@ -1,6 +1,6 @@
 {
-    "FOREMAN_SERVER": "https://foo.invalid",
-    "FOREMAN_USER": "fooo",
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "FOREMAN_PASSWORD": "fooo",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "FOREMAN_SERVER": "https://foo.invalid",
+    "FOREMAN_USER": "fooo"
 }

awx/main/tests/data/inventory/plugins/tower/env.json

@@ -1,7 +1,7 @@
 {
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "TOWER_HOST": "https://foo.invalid",
-    "TOWER_USERNAME": "fooo",
     "TOWER_PASSWORD": "fooo",
-    "TOWER_VERIFY_SSL": "False",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "TOWER_USERNAME": "fooo",
+    "TOWER_VERIFY_SSL": "False"
 }

awx/main/tests/data/inventory/scripts/azure_rm/env.json

@@ -1,9 +1,9 @@
 {
-    "AZURE_SUBSCRIPTION_ID": "fooo",
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "AZURE_CLIENT_ID": "fooo",
-    "AZURE_TENANT": "fooo",
-    "AZURE_SECRET": "fooo",
     "AZURE_CLOUD_ENVIRONMENT": "fooo",
     "AZURE_INI_PATH": "{{ file_reference }}",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "AZURE_SECRET": "fooo",
+    "AZURE_SUBSCRIPTION_ID": "fooo",
+    "AZURE_TENANT": "fooo"
 }

awx/main/tests/data/inventory/scripts/cloudforms/env.json

@@ -1,4 +1,4 @@
 {
-    "CLOUDFORMS_INI_PATH": "{{ file_reference }}",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
+    "CLOUDFORMS_INI_PATH": "{{ file_reference }}"
 }

awx/main/tests/data/inventory/scripts/ec2/env.json

@@ -1,7 +1,7 @@
 {
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "AWS_ACCESS_KEY_ID": "fooo",
     "AWS_SECRET_ACCESS_KEY": "fooo",
     "AWS_SECURITY_TOKEN": "fooo",
-    "EC2_INI_PATH": "{{ file_reference }}",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "EC2_INI_PATH": "{{ file_reference }}"
 }

awx/main/tests/data/inventory/scripts/gce/env.json

@@ -1,8 +1,12 @@
 {
-    "GCE_EMAIL": "fooo",
-    "GCE_PROJECT": "fooo",
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "GCE_CREDENTIALS_FILE_PATH": "{{ file_reference }}",
+    "GCE_EMAIL": "fooo",
+    "GCE_INI_PATH": "{{ file_reference_0 }}",
+    "GCE_PROJECT": "fooo",
     "GCE_ZONE": "us-east4-a,us-west1-b",
-    "GCE_INI_PATH": "{{ file_reference }}",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "GCP_AUTH_KIND": "serviceaccount",
+    "GCP_ENV_TYPE": "tower",
+    "GCP_PROJECT": "fooo",
+    "GCP_SERVICE_ACCOUNT_FILE": "{{ file_reference }}"
 }

awx/main/tests/data/inventory/scripts/openstack/env.json

@@ -1,4 +1,4 @@
 {
-    "OS_CLIENT_CONFIG_FILE": "{{ file_reference }}",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
+    "OS_CLIENT_CONFIG_FILE": "{{ file_reference }}"
 }

awx/main/tests/data/inventory/scripts/rhv/env.json

@@ -1,7 +1,7 @@
 {
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "OVIRT_INI_PATH": "{{ file_reference }}",
-    "OVIRT_URL": "https://foo.invalid",
-    "OVIRT_USERNAME": "fooo",
     "OVIRT_PASSWORD": "fooo",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "OVIRT_URL": "https://foo.invalid",
+    "OVIRT_USERNAME": "fooo"
 }

awx/main/tests/data/inventory/scripts/satellite6/env.json

@@ -1,4 +1,4 @@
 {
-    "FOREMAN_INI_PATH": "{{ file_reference }}",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
+    "FOREMAN_INI_PATH": "{{ file_reference }}"
 }

awx/main/tests/data/inventory/scripts/tower/env.json

@@ -1,9 +1,9 @@
 {
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "TOWER_HOST": "https://foo.invalid",
-    "TOWER_USERNAME": "fooo",
-    "TOWER_PASSWORD": "fooo",
-    "TOWER_VERIFY_SSL": "False",
     "TOWER_INVENTORY": "42",
     "TOWER_LICENSE_TYPE": "open",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "TOWER_PASSWORD": "fooo",
+    "TOWER_USERNAME": "fooo",
+    "TOWER_VERIFY_SSL": "False"
 }

awx/main/tests/data/inventory/scripts/vmware/env.json

@@ -1,8 +1,8 @@
 {
-    "VMWARE_USER": "fooo",
-    "VMWARE_PASSWORD": "fooo",
+    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never",
     "VMWARE_HOST": "https://foo.invalid",
-    "VMWARE_VALIDATE_CERTS": "False",
     "VMWARE_INI_PATH": "{{ file_reference }}",
-    "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never"
+    "VMWARE_PASSWORD": "fooo",
+    "VMWARE_USER": "fooo",
+    "VMWARE_VALIDATE_CERTS": "False"
 }

awx/main/tests/docs/test_swagger_generation.py

@@ -91,7 +91,7 @@ class TestSwaggerGeneration():
         # The number of API endpoints changes over time, but let's just check
         # for a reasonable number here; if this test starts failing, raise/lower the bounds
         paths = JSON['paths']
-        assert 250 < len(paths) < 300
+        assert 250 < len(paths) < 350
         assert list(paths['/api/'].keys()) == ['get']
         assert list(paths['/api/v2/'].keys()) == ['get']
         assert list(sorted(

awx/main/tests/functional/api/test_instance_group.py

@@ -101,7 +101,7 @@ def test_instance_group_is_isolated(instance_group, isolated_instance_group):
     assert not instance_group.is_isolated
     assert isolated_instance_group.is_isolated
 
-    isolated_instance_group.instances = []
+    isolated_instance_group.instances.set([])
 
     assert isolated_instance_group.is_isolated
 

awx/main/tests/functional/api/test_inventory.py

@@ -42,10 +42,6 @@ def test_inventory_source_notification_on_cloud_only(get, post, inventory_source
 
     not_is = inventory_source_factory("not_ec2")
 
-    url = reverse('api:inventory_source_notification_templates_any_list', kwargs={'pk': cloud_is.id})
-    response = post(url, dict(id=notification_template.id), u)
-    assert response.status_code == 204
-
     url = reverse('api:inventory_source_notification_templates_success_list', kwargs={'pk': not_is.id})
     response = post(url, dict(id=notification_template.id), u)
     assert response.status_code == 400

awx/main/tests/functional/api/test_notifications.py

@@ -0,0 +1,129 @@
+import pytest
+
+from awx.api.versioning import reverse
+
+
+@pytest.mark.django_db
+def test_get_org_running_notification(get, admin, organization):
+    url = reverse('api:organization_notification_templates_started_list', kwargs={'pk': organization.pk})
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 0
+
+
+@pytest.mark.django_db
+def test_post_org_running_notification(get, post, admin, notification_template, organization):
+    url = reverse('api:organization_notification_templates_started_list', kwargs={'pk': organization.pk})
+    response = post(url,
+                    dict(id=notification_template.id,
+                         associate=True),
+                    admin)
+    assert response.status_code == 204
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 1
+
+
+@pytest.mark.django_db
+def test_get_project_running_notification(get, admin, project):
+    url = reverse('api:project_notification_templates_started_list', kwargs={'pk': project.pk})
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 0
+
+
+@pytest.mark.django_db
+def test_post_project_running_notification(get, post, admin, notification_template, project):
+    url = reverse('api:project_notification_templates_started_list', kwargs={'pk': project.pk})
+    response = post(url,
+                    dict(id=notification_template.id,
+                         associate=True),
+                    admin)
+    assert response.status_code == 204
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 1
+
+
+@pytest.mark.django_db
+def test_get_inv_src_running_notification(get, admin, inventory_source):
+    url = reverse('api:inventory_source_notification_templates_started_list', kwargs={'pk': inventory_source.pk})
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 0
+
+
+@pytest.mark.django_db
+def test_post_inv_src_running_notification(get, post, admin, notification_template, inventory_source):
+    url = reverse('api:inventory_source_notification_templates_started_list', kwargs={'pk': inventory_source.pk})
+    response = post(url,
+                    dict(id=notification_template.id,
+                         associate=True),
+                    admin)
+    assert response.status_code == 204
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 1
+
+
+@pytest.mark.django_db
+def test_get_jt_running_notification(get, admin, job_template):
+    url = reverse('api:job_template_notification_templates_started_list', kwargs={'pk': job_template.pk})
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 0
+
+
+@pytest.mark.django_db
+def test_post_jt_running_notification(get, post, admin, notification_template, job_template):
+    url = reverse('api:job_template_notification_templates_started_list', kwargs={'pk': job_template.pk})
+    response = post(url,
+                    dict(id=notification_template.id,
+                         associate=True),
+                    admin)
+    assert response.status_code == 204
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 1
+
+
+@pytest.mark.django_db
+def test_get_sys_jt_running_notification(get, admin, system_job_template):
+    url = reverse('api:system_job_template_notification_templates_started_list', kwargs={'pk': system_job_template.pk})
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 0
+
+
+@pytest.mark.django_db
+def test_post_sys_jt_running_notification(get, post, admin, notification_template, system_job_template):
+    url = reverse('api:system_job_template_notification_templates_started_list', kwargs={'pk': system_job_template.pk})
+    response = post(url,
+                    dict(id=notification_template.id,
+                         associate=True),
+                    admin)
+    assert response.status_code == 204
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 1
+
+
+@pytest.mark.django_db
+def test_get_wfjt_running_notification(get, admin, workflow_job_template):
+    url = reverse('api:workflow_job_template_notification_templates_started_list', kwargs={'pk': workflow_job_template.pk})
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 0
+
+
+@pytest.mark.django_db
+def test_post_wfjt_running_notification(get, post, admin, notification_template, workflow_job_template):
+    url = reverse('api:workflow_job_template_notification_templates_started_list', kwargs={'pk': workflow_job_template.pk})
+    response = post(url,
+                    dict(id=notification_template.id,
+                         associate=True),
+                    admin)
+    assert response.status_code == 204
+    response = get(url, admin)
+    assert response.status_code == 200
+    assert len(response.data['results']) == 1

awx/main/tests/functional/commands/test_commands.py

@@ -12,7 +12,6 @@ def run_command(name, *args, **options):
     command_runner = options.pop('command_runner', call_command)
     stdin_fileobj = options.pop('stdin_fileobj', None)
     options.setdefault('verbosity', 1)
-    options.setdefault('interactive', False)
     original_stdin = sys.stdin
     original_stdout = sys.stdout
     original_stderr = sys.stderr

awx/main/tests/functional/conftest.py

@@ -11,9 +11,9 @@ from django.urls import resolve
 from django.contrib.auth.models import User
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db.backends.sqlite3.base import SQLiteCursorWrapper
-from jsonbfield.fields import JSONField
 
 # AWX
+from awx.main.fields import JSONBField
 from awx.main.models.projects import Project
 from awx.main.models.ha import Instance
 
@@ -737,7 +737,7 @@ def get_db_prep_save(self, value, connection, **kwargs):
 
 @pytest.fixture
 def monkeypatch_jsonbfield_get_db_prep_save(mocker):
-    JSONField.get_db_prep_save = get_db_prep_save
+    JSONBField.get_db_prep_save = get_db_prep_save
 
 
 @pytest.fixture

awx/main/tests/functional/models/test_schedule.py

@@ -457,4 +457,4 @@ def test_duplicate_name_within_template(job_template):
     with pytest.raises(IntegrityError) as ierror:
         s2.save()
 
-    assert str(ierror.value) == "columns unified_job_template_id, name are not unique"
+    assert str(ierror.value) == "UNIQUE constraint failed: main_schedule.unified_job_template_id, main_schedule.name"

awx/main/tests/functional/models/test_workflow.py

@@ -20,7 +20,6 @@ from django.test import TransactionTestCase
 from django.core.exceptions import ValidationError
 
 
-@pytest.mark.django_db
 class TestWorkflowDAGFunctional(TransactionTestCase):
     def workflow_job(self, states=['new', 'new', 'new', 'new', 'new']):
         """

awx/main/tests/functional/task_management/test_capacity.py

@@ -1,5 +1,3 @@
-import pytest
-
 from django.test import TransactionTestCase
 
 from awx.main.models import (
@@ -8,7 +6,6 @@ from awx.main.models import (
 )
 
 
-@pytest.mark.django_db
 class TestCapacityMapping(TransactionTestCase):
 
     def sample_cluster(self):

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -129,6 +129,7 @@ def read_content(private_data_dir, raw_env, inventory_update):
     return a dictionary `content` with file contents, keyed off environment variable
         that references the file
     """
+    # build dict env as a mapping of environment variables to file names
     # Filter out environment variables which come from runtime environment
     env = {}
     exclude_keys = set(('PATH', 'INVENTORY_SOURCE_ID', 'INVENTORY_UPDATE_ID'))
@@ -142,71 +143,74 @@ def read_content(private_data_dir, raw_env, inventory_update):
             env[k] = v
     inverse_env = {}
     for key, value in env.items():
-        inverse_env[value] = key
+        inverse_env.setdefault(value, []).append(key)
 
     cache_file_regex = re.compile(r'/tmp/awx_{0}_[a-zA-Z0-9_]+/{1}_cache[a-zA-Z0-9_]+'.format(
         inventory_update.id, inventory_update.source)
     )
     private_key_regex = re.compile(r'-----BEGIN ENCRYPTED PRIVATE KEY-----.*-----END ENCRYPTED PRIVATE KEY-----')
 
+    # read directory content
+    # build a mapping of the file paths to aliases which will be constant accross runs
     dir_contents = {}
-    references = {}
-    for filename in os.listdir(private_data_dir):
+    referenced_paths = set()
+    file_aliases = {}
+    filename_list = sorted(os.listdir(private_data_dir), key=lambda fn: inverse_env.get(os.path.join(private_data_dir, fn), [fn])[0])
+    for filename in filename_list:
         if filename in ('args', 'project'):
             continue  # Ansible runner
         abs_file_path = os.path.join(private_data_dir, filename)
+        file_aliases[abs_file_path] = filename
         if abs_file_path in inverse_env:
-            env_key = inverse_env[abs_file_path]
-            references[abs_file_path] = env_key
-            env[env_key] = '{{ file_reference }}'
+            referenced_paths.add(abs_file_path)
+            alias = 'file_reference'
+            for i in range(10):
+                if alias not in file_aliases.values():
+                    break
+                alias = 'file_reference_{}'.format(i)
+            else:
+                raise RuntimeError('Test not able to cope with >10 references by env vars. '
+                                   'Something probably went very wrong.')
+            file_aliases[abs_file_path] = alias
+            for env_key in inverse_env[abs_file_path]:
+                env[env_key] = '{{{{ {} }}}}'.format(alias)
         try:
             with open(abs_file_path, 'r') as f:
                 dir_contents[abs_file_path] = f.read()
             # Declare a reference to inventory plugin file if it exists
             if abs_file_path.endswith('.yml') and 'plugin: ' in dir_contents[abs_file_path]:
-                references[abs_file_path] = filename  # plugin filenames are universal
+                referenced_paths.add(abs_file_path)  # used as inventory file
+            elif cache_file_regex.match(abs_file_path):
+                file_aliases[abs_file_path] = 'cache_file'
         except IsADirectoryError:
             dir_contents[abs_file_path] = '<directory>'
+            if cache_file_regex.match(abs_file_path):
+                file_aliases[abs_file_path] = 'cache_dir'
 
-    # Declare cross-file references, also use special keywords if it is the cache
-    cache_referenced = False
-    cache_present = False
+    # Substitute in aliases for cross-file references
     for abs_file_path, file_content in dir_contents.copy().items():
         if cache_file_regex.match(file_content):
-            cache_referenced = True
+            if 'cache_dir' not in file_aliases.values() and 'cache_file' not in file_aliases in file_aliases.values():
+                raise AssertionError(
+                    'A cache file was referenced but never created, files:\n{}'.format(
+                        json.dumps(dir_contents, indent=4)))
+        # if another files path appears in this file, replace it with its alias
         for target_path in dir_contents.keys():
+            other_alias = file_aliases[target_path]
             if target_path in file_content:
-                if target_path in references:
-                    raise AssertionError(
-                        'File {} is referenced by env var or other file as well as file {}:\n{}\n{}'.format(
-                            target_path, abs_file_path, json.dumps(env, indent=4), json.dumps(dir_contents, indent=4)))
-                else:
-                    if cache_file_regex.match(target_path):
-                        cache_present = True
-                        if os.path.isdir(target_path):
-                            keyword = 'cache_dir'
-                        else:
-                            keyword = 'cache_file'
-                        references[target_path] = keyword
-                        new_file_content = cache_file_regex.sub('{{ ' + keyword + ' }}', file_content)
-                    else:
-                        references[target_path] = 'file_reference'
-                        new_file_content = file_content.replace(target_path, '{{ file_reference }}')
-                    dir_contents[abs_file_path] = new_file_content
-    if cache_referenced and not cache_present:
-        raise AssertionError(
-            'A cache file was referenced but never created, files:\n{}'.format(
-                json.dumps(dir_contents, indent=4)))
+                referenced_paths.add(target_path)
+                dir_contents[abs_file_path] = file_content.replace(target_path, '{{ ' + other_alias + ' }}')
 
+    # build dict content which is the directory contents keyed off the file aliases
     content = {}
     for abs_file_path, file_content in dir_contents.items():
-        if abs_file_path not in references:
+        # assert that all files laid down are used
+        if abs_file_path not in referenced_paths:
             raise AssertionError(
                 "File {} is not referenced. References and files:\n{}\n{}".format(
-                    abs_file_path, json.dumps(references, indent=4), json.dumps(dir_contents, indent=4)))
-        reference_key = references[abs_file_path]
+                    abs_file_path, json.dumps(env, indent=4), json.dumps(dir_contents, indent=4)))
         file_content = private_key_regex.sub('{{private_key}}', file_content)
-        content[reference_key] = file_content
+        content[file_aliases[abs_file_path]] = file_content
 
     return (env, content)
 
@@ -223,7 +227,7 @@ def create_reference_data(source_dir, env, content):
                 f.write(content)
     if env:
         with open(os.path.join(source_dir, 'env.json'), 'w') as f:
-            f.write(json.dumps(env, indent=4))
+            json.dump(env, f, indent=4, sort_keys=True)
 
 
 @pytest.mark.django_db
@@ -258,6 +262,7 @@ def test_inventory_update_injected_content(this_kind, script_or_plugin, inventor
         """
         private_data_dir = envvars.pop('AWX_PRIVATE_DATA_DIR')
         assert envvars.pop('ANSIBLE_INVENTORY_ENABLED') == ('auto' if use_plugin else 'script')
+        assert envvars.pop('ANSIBLE_COLLECTIONS_PATHS') == os.pathsep.join(settings.AWX_ANSIBLE_COLLECTIONS_PATHS)
         set_files = bool(os.getenv("MAKE_INVENTORY_REFERENCE_FILES", 'false').lower()[0] not in ['f', '0'])
         env, content = read_content(private_data_dir, envvars, inventory_update)
         base_dir = os.path.join(DATA, script_or_plugin)
@@ -283,7 +288,7 @@ def test_inventory_update_injected_content(this_kind, script_or_plugin, inventor
             for f_name in expected_file_list:
                 with open(os.path.join(files_dir, f_name), 'r') as f:
                     ref_content = f.read()
-                    assert ref_content == content[f_name]
+                    assert ref_content == content[f_name], f_name
             try:
                 with open(os.path.join(source_dir, 'env.json'), 'r') as f:
                     ref_env_text = f.read()

awx/main/tests/functional/test_jobs.py

@@ -42,7 +42,7 @@ def test_job_notification_data(inventory, machine_credential, project):
         survey_passwords={"SSN": encrypted_str},
         project=project,
     )
-    job.credentials = [machine_credential]
+    job.credentials.set([machine_credential])
     notification_data = job.notification_data(block=0)
     assert json.loads(notification_data['extra_vars'])['SSN'] == encrypted_str
 

awx/main/tests/functional/test_notifications.py

@@ -92,27 +92,7 @@ def test_inherited_notification_templates(get, post, user, organization, project
     isrc.save()
     jt = JobTemplate.objects.create(name='test', inventory=i, project=project, playbook='debug.yml')
     jt.save()
-    url = reverse('api:organization_notification_templates_any_list', kwargs={'pk': organization.id})
-    response = post(url, dict(id=notification_templates[0]), u)
-    assert response.status_code == 204
-    url = reverse('api:project_notification_templates_any_list', kwargs={'pk': project.id})
-    response = post(url, dict(id=notification_templates[1]), u)
-    assert response.status_code == 204
-    url = reverse('api:job_template_notification_templates_any_list', kwargs={'pk': jt.id})
-    response = post(url, dict(id=notification_templates[2]), u)
-    assert response.status_code == 204
-    assert len(jt.notification_templates['any']) == 3
-    assert len(project.notification_templates['any']) == 2
-    assert len(isrc.notification_templates['any']) == 1
-
-
-@pytest.mark.django_db
-def test_notification_template_merging(get, post, user, organization, project, notification_template):
-    user('admin-poster', True)
-    organization.notification_templates_any.add(notification_template)
-    project.notification_templates_any.add(notification_template)
-    assert len(project.notification_templates['any']) == 1
-
+    
 
 @pytest.mark.django_db
 def test_notification_template_simple_patch(patch, notification_template, admin):

awx/main/tests/functional/test_rbac_user.py

@@ -7,7 +7,6 @@ from awx.main.access import UserAccess, RoleAccess, TeamAccess
 from awx.main.models import User, Organization, Inventory
 
 
-@pytest.mark.django_db
 class TestSysAuditorTransactional(TransactionTestCase):
     def rando(self):
         return User.objects.create(username='rando', password='rando', email='rando@com.com')

awx/main/tests/unit/api/serializers/test_inventory_serializers.py

@@ -71,7 +71,7 @@ class TestInventorySourceSerializerGetRelated(object):
         'activity_stream',
         'notification_templates_error',
         'notification_templates_success',
-        'notification_templates_any',
+        'notification_templates_started',
         'inventory_updates',
         'update',
         'hosts',

awx/main/tests/unit/api/serializers/test_job_template_serializers.py

@@ -50,7 +50,7 @@ class TestJobTemplateSerializerGetRelated():
         'schedules',
         'activity_stream',
         'launch',
-        'notification_templates_any',
+        'notification_templates_started',
         'notification_templates_success',
         'notification_templates_error',
         'survey_spec',

awx/main/tests/unit/api/test_generics.py

@@ -186,11 +186,8 @@ class TestResourceAccessList:
 
     def mock_request(self):
         return mock.MagicMock(
-            user=mock.MagicMock(
-                is_anonymous=mock.MagicMock(return_value=False),
-                is_superuser=False
-            ), method='GET')
-
+            user=mock.MagicMock(is_anonymous=False, is_superuser=False),
+            method='GET')
 
     def mock_view(self, parent=None):
         view = ResourceAccessList()
@@ -200,7 +197,6 @@ class TestResourceAccessList:
             view.get_parent_object = lambda: parent
         return view
 
-
     def test_parent_access_check_failed(self, mocker, mock_organization):
         mock_access = mocker.MagicMock(__name__='for logger', return_value=False)
         with mocker.patch('awx.main.access.BaseAccess.can_read', mock_access):
@@ -208,7 +204,6 @@ class TestResourceAccessList:
                 self.mock_view(parent=mock_organization).check_permissions(self.mock_request())
             mock_access.assert_called_once_with(mock_organization)
 
-
     def test_parent_access_check_worked(self, mocker, mock_organization):
         mock_access = mocker.MagicMock(__name__='for logger', return_value=True)
         with mocker.patch('awx.main.access.BaseAccess.can_read', mock_access):

awx/main/tests/unit/conftest.py

@@ -6,7 +6,7 @@ from unittest.mock import PropertyMock
 from awx.api.urls import urlpatterns as api_patterns
 
 # Django
-from django.urls import RegexURLResolver, RegexURLPattern
+from django.urls import URLResolver, URLPattern
 
 
 @pytest.fixture(autouse=True)
@@ -20,24 +20,24 @@ def all_views():
     '''
     returns a set of all views in the app
     '''
-    patterns = set([])
-    url_views = set([])
+    patterns = set()
+    url_views = set()
     # Add recursive URL patterns
     unprocessed = set(api_patterns)
     while unprocessed:
         to_process = unprocessed.copy()
-        unprocessed = set([])
+        unprocessed = set()
         for pattern in to_process:
             if hasattr(pattern, 'lookup_str') and not pattern.lookup_str.startswith('awx.api'):
                 continue
             patterns.add(pattern)
-            if isinstance(pattern, RegexURLResolver):
+            if isinstance(pattern, URLResolver):
                 for sub_pattern in pattern.url_patterns:
                     if sub_pattern not in patterns:
                         unprocessed.add(sub_pattern)
     # Get view classes
     for pattern in patterns:
-        if isinstance(pattern, RegexURLPattern) and hasattr(pattern.callback, 'view_class'):
+        if isinstance(pattern, URLPattern) and hasattr(pattern.callback, 'view_class'):
             url_views.add(pattern.callback.view_class)
     return url_views
 

awx/main/tests/unit/models/test_events.py

@@ -57,5 +57,5 @@ def test_really_long_event_fields(field):
         })
         manager.create.assert_called_with(**{
             'job_id': 123,
-            'event_data': {field: 'X' * 1021 + '...'}
+            'event_data': {field: 'X' * 1023 + '…'}
         })

awx/main/tests/unit/models/test_unified_job_unit.py

@@ -9,7 +9,8 @@ from awx.main.models import (
     Job,
     User,
     Project,
-    JobTemplate
+    JobTemplate,
+    Inventory
 )
 
 
@@ -81,11 +82,13 @@ class TestMetaVars:
 
     def test_job_metavars(self):
         maker = User(username='joe', pk=47, id=47)
+        inv = Inventory(name='example-inv', id=45)
         assert Job(
             name='fake-job',
             pk=42, id=42,
             launch_type='manual',
-            created_by=maker
+            created_by=maker,
+            inventory=inv
         ).awx_meta_vars() == {
             'tower_job_id': 42,
             'awx_job_id': 42,
@@ -100,7 +103,11 @@ class TestMetaVars:
             'awx_user_last_name': '',
             'tower_user_last_name': '',
             'awx_user_id': 47,
-            'tower_user_id': 47
+            'tower_user_id': 47,
+            'tower_inventory_id': 45,
+            'awx_inventory_id': 45,
+            'tower_inventory_name': 'example-inv',
+            'awx_inventory_name': 'example-inv'
         }
 
     def test_project_update_metavars(self):