Add example for ad_hoc_command module (#14106)

When we close/cancel a connection to a web node, give the task time to
clean up after itself and cleanly exit. Otherwise, the Python GC might
clean up the task too early and this leads to ugly log messages like
this: "Task was destroyed but it is pending!"

Signed-off-by: Rick Elrod <rick@elrod.me>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -19,6 +19,8 @@ body:
           required: true
         - label: I understand that AWX is open source software provided for free and that I might not receive a timely response.
           required: true
+        - label: I am **NOT** reporting a (potential) security vulnerability. (These should be emailed to `security@ansible.com` instead.)
+          required: true
 
   - type: textarea
     id: summary
@@ -42,6 +44,7 @@ body:
       label: Select the relevant components
       options:
         - label: UI
+        - label: UI (tech preview)
         - label: API
         - label: Docs
         - label: Collection

.github/dependabot.yml

@@ -1,19 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "npm"
-    directory: "/awx/ui"
-    schedule:
-      interval: "monthly"
-    open-pull-requests-limit: 5
-    allow:
-      - dependency-type: "production"
-    reviewers:
-      - "AlexSCorey"
-      - "keithjgrant"
-      - "kialam"
-      - "mabashian"
-      - "marshmalien"
-    labels:
-      - "component:ui"
-      - "dependencies"
-    target-branch: "devel"

.github/issue_labeler.yml

@@ -6,6 +6,8 @@ needs_triage:
   - "Feature Summary"
 "component:ui":
   - "\\[X\\] UI"
+"component:ui_next":
+  - "\\[X\\] UI \\(tech preview\\)"
 "component:api":
   - "\\[X\\] API"
 "component:docs":

.github/workflows/label_issue.yml

@@ -6,9 +6,9 @@ on:
       - opened
       - reopened
 
-    permissions:
-      contents: read # to fetch code
-      issues: write # to label issues
+permissions:
+  contents: write # to fetch code
+  issues: write # to label issues
 
 jobs:
   triage:

.github/workflows/label_pr.yml

@@ -8,7 +8,7 @@ on:
       - synchronize
 
 permissions:
-  contents: read # to determine modified files (actions/labeler)
+  contents: write # to determine modified files (actions/labeler)
   pull-requests: write # to add labels to PRs (actions/labeler)
 
 jobs:

ISSUES.md

@@ -31,7 +31,7 @@ If your issue isn't considered high priority, then please be patient as it may t
 
 `state:needs_info` The issue needs more information. This could be more debug output, more specifics out the system such as version information. Any detail that is currently preventing this issue from moving forward. This should be considered a blocked state.
 
-`state:needs_review` The issue/pull request needs to be reviewed by other maintainers and contributors. This is usually used when there is a question out to another maintainer or when a person is less familar with an area of the code base the issue is for.
+`state:needs_review` The issue/pull request needs to be reviewed by other maintainers and contributors. This is usually used when there is a question out to another maintainer or when a person is less familiar with an area of the code base the issue is for.
 
 `state:needs_revision` More commonly used on pull requests, this state represents that there are changes that are being waited on.
 

Makefile

@@ -8,7 +8,7 @@ NPM_BIN ?= npm
 CHROMIUM_BIN=/tmp/chrome-linux/chrome
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 MANAGEMENT_COMMAND ?= awx-manage
-VERSION := $(shell $(PYTHON) tools/scripts/scm_version.py)
+VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py)
 
 # ansible-test requires semver compatable version, so we allow overrides to hack it
 COLLECTION_VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py | cut -d . -f 1-3)
@@ -52,7 +52,7 @@ RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
 
 # Python packages to install only from source (not from binary wheels)
 # Comma separated list
-SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio
+SRC_ONLY_PKGS ?= cffi,pycparser,psycopg,twilio
 # These should be upgraded in the AWX and Ansible venv before attempting
 # to install the actual requirements
 VENV_BOOTSTRAP ?= pip==21.2.4 setuptools==65.6.3 setuptools_scm[toml]==7.0.5 wheel==0.38.4
@@ -267,11 +267,11 @@ run-wsrelay:
 	$(PYTHON) manage.py run_wsrelay
 
 ## Start the heartbeat process in background in development environment.
-run-heartbeet:
+run-ws-heartbeat:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	$(PYTHON) manage.py run_heartbeet
+	$(PYTHON) manage.py run_ws_heartbeat
 
 reports:
 	mkdir -p $@
@@ -660,10 +660,12 @@ awx-kube-dev-build: Dockerfile.kube-dev
 ## generate UI .pot file, an empty template of strings yet to be translated
 pot: $(UI_BUILD_FLAG_FILE)
 	$(NPM_BIN) --prefix awx/ui --loglevel warn run extract-template --clean
+	$(NPM_BIN) --prefix awx/ui_next --loglevel warn run extract-template --clean
 
 ## generate UI .po files for each locale (will update translated strings for `en`)
 po: $(UI_BUILD_FLAG_FILE)
 	$(NPM_BIN) --prefix awx/ui --loglevel warn run extract-strings -- --clean
+	$(NPM_BIN) --prefix awx/ui_next --loglevel warn run extract-strings -- --clean
 
 ## generate API django .pot .po
 messages:

awx/api/filters.py

@@ -347,7 +347,7 @@ class FieldLookupBackend(BaseFilterBackend):
                         args.append(Q(**{k: v}))
                 for role_name in role_filters:
                     if not hasattr(queryset.model, 'accessible_pk_qs'):
-                        raise ParseError(_('Cannot apply role_level filter to this list because its model ' 'does not use roles for access control.'))
+                        raise ParseError(_('Cannot apply role_level filter to this list because its model does not use roles for access control.'))
                     args.append(Q(pk__in=queryset.model.accessible_pk_qs(request.user, role_name)))
                 if or_filters:
                     q = Q()

awx/api/generics.py

@@ -169,7 +169,7 @@ class APIView(views.APIView):
             self.__init_request_error__ = exc
         except UnsupportedMediaType as exc:
             exc.detail = _(
-                'You did not use correct Content-Type in your HTTP request. ' 'If you are using our REST API, the Content-Type must be application/json'
+                'You did not use correct Content-Type in your HTTP request. If you are using our REST API, the Content-Type must be application/json'
             )
             self.__init_request_error__ = exc
         return drf_request

awx/api/metadata.py

@@ -71,7 +71,7 @@ class Metadata(metadata.SimpleMetadata):
                 'url': _('URL for this {}.'),
                 'related': _('Data structure with URLs of related resources.'),
                 'summary_fields': _(
-                    'Data structure with name/description for related resources.  ' 'The output for some objects may be limited for performance reasons.'
+                    'Data structure with name/description for related resources.  The output for some objects may be limited for performance reasons.'
                 ),
                 'created': _('Timestamp when this {} was created.'),
                 'modified': _('Timestamp when this {} was last modified.'),

awx/api/serializers.py

@@ -220,7 +220,7 @@ class CopySerializer(serializers.Serializer):
         view = self.context.get('view', None)
         obj = view.get_object()
         if name == obj.name:
-            raise serializers.ValidationError(_('The original object is already named {}, a copy from' ' it cannot have the same name.'.format(name)))
+            raise serializers.ValidationError(_('The original object is already named {}, a copy from it cannot have the same name.'.format(name)))
         return attrs
 
 
@@ -760,7 +760,7 @@ class UnifiedJobTemplateSerializer(BaseSerializer):
 class UnifiedJobSerializer(BaseSerializer):
     show_capabilities = ['start', 'delete']
     event_processing_finished = serializers.BooleanField(
-        help_text=_('Indicates whether all of the events generated by this ' 'unified job have been saved to the database.'), read_only=True
+        help_text=_('Indicates whether all of the events generated by this unified job have been saved to the database.'), read_only=True
     )
 
     class Meta:
@@ -1579,7 +1579,7 @@ class ProjectPlaybooksSerializer(ProjectSerializer):
 
 
 class ProjectInventoriesSerializer(ProjectSerializer):
-    inventory_files = serializers.ReadOnlyField(help_text=_('Array of inventory files and directories available within this project, ' 'not comprehensive.'))
+    inventory_files = serializers.ReadOnlyField(help_text=_('Array of inventory files and directories available within this project, not comprehensive.'))
 
     class Meta:
         model = Project
@@ -2905,7 +2905,7 @@ class CredentialSerializer(BaseSerializer):
             ):
                 if getattr(self.instance, related_objects).count() > 0:
                     raise ValidationError(
-                        _('You cannot change the credential type of the credential, as it may break the functionality' ' of the resources using it.')
+                        _('You cannot change the credential type of the credential, as it may break the functionality of the resources using it.')
                     )
 
         return credential_type
@@ -2925,7 +2925,7 @@ class CredentialSerializerCreate(CredentialSerializer):
         default=None,
         write_only=True,
         allow_null=True,
-        help_text=_('Write-only field used to add user to owner role. If provided, ' 'do not give either team or organization. Only valid for creation.'),
+        help_text=_('Write-only field used to add user to owner role. If provided, do not give either team or organization. Only valid for creation.'),
     )
     team = serializers.PrimaryKeyRelatedField(
         queryset=Team.objects.all(),
@@ -2933,14 +2933,14 @@ class CredentialSerializerCreate(CredentialSerializer):
         default=None,
         write_only=True,
         allow_null=True,
-        help_text=_('Write-only field used to add team to owner role. If provided, ' 'do not give either user or organization. Only valid for creation.'),
+        help_text=_('Write-only field used to add team to owner role. If provided, do not give either user or organization. Only valid for creation.'),
     )
     organization = serializers.PrimaryKeyRelatedField(
         queryset=Organization.objects.all(),
         required=False,
         default=None,
         allow_null=True,
-        help_text=_('Inherit permissions from organization roles. If provided on creation, ' 'do not give either user or team.'),
+        help_text=_('Inherit permissions from organization roles. If provided on creation, do not give either user or team.'),
     )
 
     class Meta:
@@ -2962,7 +2962,7 @@ class CredentialSerializerCreate(CredentialSerializer):
         if len(owner_fields) > 1:
             received = ", ".join(sorted(owner_fields))
             raise serializers.ValidationError(
-                {"detail": _("Only one of 'user', 'team', or 'organization' should be provided, " "received {} fields.".format(received))}
+                {"detail": _("Only one of 'user', 'team', or 'organization' should be provided, received {} fields.".format(received))}
             )
 
         if attrs.get('team'):
@@ -3622,7 +3622,7 @@ class SystemJobSerializer(UnifiedJobSerializer):
         try:
             return obj.result_stdout
         except StdoutMaxBytesExceeded as e:
-            return _("Standard Output too large to display ({text_size} bytes), " "only download supported for sizes over {supported_size} bytes.").format(
+            return _("Standard Output too large to display ({text_size} bytes), only download supported for sizes over {supported_size} bytes.").format(
                 text_size=e.total, supported_size=e.supported
             )
 
@@ -4536,7 +4536,7 @@ class JobLaunchSerializer(BaseSerializer):
                 if cred.unique_hash() in provided_mapping.keys():
                     continue  # User replaced credential with new of same type
                 errors.setdefault('credentials', []).append(
-                    _('Removing {} credential at launch time without replacement is not supported. ' 'Provided list lacked credential(s): {}.').format(
+                    _('Removing {} credential at launch time without replacement is not supported. Provided list lacked credential(s): {}.').format(
                         cred.unique_hash(display=True), ', '.join([str(c) for c in removed_creds])
                     )
                 )
@@ -4686,12 +4686,11 @@ class BulkJobNodeSerializer(WorkflowJobNodeSerializer):
     # many-to-many fields
     credentials = serializers.ListField(child=serializers.IntegerField(min_value=1), required=False)
     labels = serializers.ListField(child=serializers.IntegerField(min_value=1), required=False)
-    # TODO: Use instance group role added via PR 13584(once merged), for now everything related to instance group is commented
-    # instance_groups = serializers.ListField(child=serializers.IntegerField(min_value=1), required=False)
+    instance_groups = serializers.ListField(child=serializers.IntegerField(min_value=1), required=False)
 
     class Meta:
         model = WorkflowJobNode
-        fields = ('*', 'credentials', 'labels')  # m2m fields are not canonical for WJ nodes, TODO: add instance_groups once supported
+        fields = ('*', 'credentials', 'labels', 'instance_groups')  # m2m fields are not canonical for WJ nodes
 
     def validate(self, attrs):
         return super(LaunchConfigurationBaseSerializer, self).validate(attrs)
@@ -4751,21 +4750,21 @@ class BulkJobLaunchSerializer(serializers.Serializer):
         requested_use_execution_environments = {job['execution_environment'] for job in attrs['jobs'] if 'execution_environment' in job}
         requested_use_credentials = set()
         requested_use_labels = set()
-        # requested_use_instance_groups = set()
+        requested_use_instance_groups = set()
         for job in attrs['jobs']:
             for cred in job.get('credentials', []):
                 requested_use_credentials.add(cred)
             for label in job.get('labels', []):
                 requested_use_labels.add(label)
-            # for instance_group in job.get('instance_groups', []):
-            #     requested_use_instance_groups.add(instance_group)
+            for instance_group in job.get('instance_groups', []):
+                requested_use_instance_groups.add(instance_group)
 
         key_to_obj_map = {
             "unified_job_template": {obj.id: obj for obj in UnifiedJobTemplate.objects.filter(id__in=requested_ujts)},
             "inventory": {obj.id: obj for obj in Inventory.objects.filter(id__in=requested_use_inventories)},
             "credentials": {obj.id: obj for obj in Credential.objects.filter(id__in=requested_use_credentials)},
             "labels": {obj.id: obj for obj in Label.objects.filter(id__in=requested_use_labels)},
-            # "instance_groups": {obj.id: obj for obj in InstanceGroup.objects.filter(id__in=requested_use_instance_groups)},
+            "instance_groups": {obj.id: obj for obj in InstanceGroup.objects.filter(id__in=requested_use_instance_groups)},
             "execution_environment": {obj.id: obj for obj in ExecutionEnvironment.objects.filter(id__in=requested_use_execution_environments)},
         }
 
@@ -4792,7 +4791,7 @@ class BulkJobLaunchSerializer(serializers.Serializer):
 
         self.check_list_permission(Credential, requested_use_credentials, 'use_role')
         self.check_list_permission(Label, requested_use_labels)
-        # self.check_list_permission(InstanceGroup, requested_use_instance_groups)  # TODO: change to use_role for conflict
+        self.check_list_permission(InstanceGroup, requested_use_instance_groups)  # TODO: change to use_role for conflict
         self.check_list_permission(ExecutionEnvironment, requested_use_execution_environments)  # TODO: change if roles introduced
 
         jobs_object = self.get_objectified_jobs(attrs, key_to_obj_map)
@@ -4839,7 +4838,7 @@ class BulkJobLaunchSerializer(serializers.Serializer):
         node_m2m_object_types_to_through_model = {
             'credentials': WorkflowJobNode.credentials.through,
             'labels': WorkflowJobNode.labels.through,
-            # 'instance_groups': WorkflowJobNode.instance_groups.through,
+            'instance_groups': WorkflowJobNode.instance_groups.through,
         }
         node_deferred_attr_names = (
             'limit',
@@ -4892,9 +4891,9 @@ class BulkJobLaunchSerializer(serializers.Serializer):
                 if field_name in node_m2m_objects[node_identifier] and field_name == 'labels':
                     for label in node_m2m_objects[node_identifier][field_name]:
                         through_model_objects.append(through_model(label=label, workflowjobnode=node_m2m_objects[node_identifier]['node']))
-                # if obj_type in node_m2m_objects[node_identifier] and obj_type == 'instance_groups':
-                #     for instance_group in node_m2m_objects[node_identifier][obj_type]:
-                #         through_model_objects.append(through_model(instancegroup=instance_group, workflowjobnode=node_m2m_objects[node_identifier]['node']))
+                if field_name in node_m2m_objects[node_identifier] and field_name == 'instance_groups':
+                    for instance_group in node_m2m_objects[node_identifier][field_name]:
+                        through_model_objects.append(through_model(instancegroup=instance_group, workflowjobnode=node_m2m_objects[node_identifier]['node']))
             if through_model_objects:
                 through_model.objects.bulk_create(through_model_objects)
 
@@ -5019,7 +5018,7 @@ class NotificationTemplateSerializer(BaseSerializer):
                     for subevent in event_messages:
                         if subevent not in ('running', 'approved', 'timed_out', 'denied'):
                             error_list.append(
-                                _("Workflow Approval event '{}' invalid, must be one of " "'running', 'approved', 'timed_out', or 'denied'").format(subevent)
+                                _("Workflow Approval event '{}' invalid, must be one of 'running', 'approved', 'timed_out', or 'denied'").format(subevent)
                             )
                             continue
                         subevent_messages = event_messages[subevent]
@@ -5436,7 +5435,7 @@ class InstanceSerializer(BaseSerializer):
         res = super(InstanceSerializer, self).get_related(obj)
         res['jobs'] = self.reverse('api:instance_unified_jobs_list', kwargs={'pk': obj.pk})
         res['instance_groups'] = self.reverse('api:instance_instance_groups_list', kwargs={'pk': obj.pk})
-        if settings.IS_K8S and obj.node_type in (Instance.Types.EXECUTION,):
+        if obj.node_type in [Instance.Types.EXECUTION, Instance.Types.HOP]:
             res['install_bundle'] = self.reverse('api:instance_install_bundle', kwargs={'pk': obj.pk})
         res['peers'] = self.reverse('api:instance_peers_list', kwargs={"pk": obj.pk})
         if self.context['request'].user.is_superuser or self.context['request'].user.is_system_auditor:
@@ -5559,7 +5558,7 @@ class InstanceGroupSerializer(BaseSerializer):
     instances = serializers.SerializerMethodField()
     is_container_group = serializers.BooleanField(
         required=False,
-        help_text=_('Indicates whether instances in this group are containerized.' 'Containerized groups have a designated Openshift or Kubernetes cluster.'),
+        help_text=_('Indicates whether instances in this group are containerized.Containerized groups have a designated Openshift or Kubernetes cluster.'),
     )
     # NOTE: help_text is duplicated from field definitions, no obvious way of
     # both defining field details here and also getting the field's help_text
@@ -5570,7 +5569,7 @@ class InstanceGroupSerializer(BaseSerializer):
         required=False,
         initial=0,
         label=_('Policy Instance Percentage'),
-        help_text=_("Minimum percentage of all instances that will be automatically assigned to " "this group when new instances come online."),
+        help_text=_("Minimum percentage of all instances that will be automatically assigned to this group when new instances come online."),
     )
     policy_instance_minimum = serializers.IntegerField(
         default=0,
@@ -5578,7 +5577,7 @@ class InstanceGroupSerializer(BaseSerializer):
         required=False,
         initial=0,
         label=_('Policy Instance Minimum'),
-        help_text=_("Static minimum number of Instances that will be automatically assign to " "this group when new instances come online."),
+        help_text=_("Static minimum number of Instances that will be automatically assign to this group when new instances come online."),
     )
     max_concurrent_jobs = serializers.IntegerField(
         default=0,

awx/api/swagger.py

@@ -1,16 +1,10 @@
-import json
 import warnings
 
-from coreapi.document import Object, Link
-
-from rest_framework import exceptions
 from rest_framework.permissions import AllowAny
-from rest_framework.renderers import CoreJSONRenderer
-from rest_framework.response import Response
 from rest_framework.schemas import SchemaGenerator, AutoSchema as DRFAuthSchema
-from rest_framework.views import APIView
 
-from rest_framework_swagger import renderers
+from drf_yasg.views import get_schema_view
+from drf_yasg import openapi
 
 
 class SuperUserSchemaGenerator(SchemaGenerator):
@@ -55,43 +49,15 @@ class AutoSchema(DRFAuthSchema):
         return description
 
 
-class SwaggerSchemaView(APIView):
-    _ignore_model_permissions = True
-    exclude_from_schema = True
-    permission_classes = [AllowAny]
-    renderer_classes = [CoreJSONRenderer, renderers.OpenAPIRenderer, renderers.SwaggerUIRenderer]
-
-    def get(self, request):
-        generator = SuperUserSchemaGenerator(title='Ansible Automation Platform controller API', patterns=None, urlconf=None)
-        schema = generator.get_schema(request=request)
-        # python core-api doesn't support the deprecation yet, so track it
-        # ourselves and return it in a response header
-        _deprecated = []
-
-        # By default, DRF OpenAPI serialization places all endpoints in
-        # a single node based on their root path (/api).  Instead, we want to
-        # group them by topic/tag so that they're categorized in the rendered
-        # output
-        document = schema._data.pop('api')
-        for path, node in document.items():
-            if isinstance(node, Object):
-                for action in node.values():
-                    topic = getattr(action, 'topic', None)
-                    if topic:
-                        schema._data.setdefault(topic, Object())
-                        schema._data[topic]._data[path] = node
-
-                    if isinstance(action, Object):
-                        for link in action.links.values():
-                            if link.deprecated:
-                                _deprecated.append(link.url)
-            elif isinstance(node, Link):
-                topic = getattr(node, 'topic', None)
-                if topic:
-                    schema._data.setdefault(topic, Object())
-                    schema._data[topic]._data[path] = node
-
-        if not schema:
-            raise exceptions.ValidationError('The schema generator did not return a schema Document')
-
-        return Response(schema, headers={'X-Deprecated-Paths': json.dumps(_deprecated)})
+schema_view = get_schema_view(
+    openapi.Info(
+        title="Snippets API",
+        default_version='v1',
+        description="Test description",
+        terms_of_service="https://www.google.com/policies/terms/",
+        contact=openapi.Contact(email="contact@snippets.local"),
+        license=openapi.License(name="BSD License"),
+    ),
+    public=True,
+    permission_classes=[AllowAny],
+)

awx/api/templates/instance_install_bundle/group_vars/all.yml

@@ -9,7 +9,7 @@ receptor_work_commands:
     params: worker
     allowruntimeparams: true
     verifysignature: true
-custom_worksign_public_keyfile: receptor/work-public-key.pem
+custom_worksign_public_keyfile: receptor/work_public_key.pem
 custom_tls_certfile: receptor/tls/receptor.crt
 custom_tls_keyfile: receptor/tls/receptor.key
 custom_ca_certfile: receptor/tls/ca/receptor-ca.crt

awx/api/urls/urls.py

@@ -167,10 +167,13 @@ urlpatterns = [
 ]
 if MODE == 'development':
     # Only include these if we are in the development environment
-    from awx.api.swagger import SwaggerSchemaView
-
-    urlpatterns += [re_path(r'^swagger/$', SwaggerSchemaView.as_view(), name='swagger_view')]
+    from awx.api.swagger import schema_view
 
     from awx.api.urls.debug import urls as debug_urls
 
     urlpatterns += [re_path(r'^debug/', include(debug_urls))]
+    urlpatterns += [
+        re_path(r'^swagger(?P<format>\.json|\.yaml)/$', schema_view.without_ui(cache_timeout=0), name='schema-json'),
+        re_path(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
+        re_path(r'^redoc/$', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'),
+    ]

awx/api/views/__init__.py

@@ -565,7 +565,7 @@ class LaunchConfigCredentialsBase(SubListAttachDetachAPIView):
         if self.relationship not in ask_mapping:
             return {"msg": _("Related template cannot accept {} on launch.").format(self.relationship)}
         elif sub.passwords_needed:
-            return {"msg": _("Credential that requires user input on launch " "cannot be used in saved launch configuration.")}
+            return {"msg": _("Credential that requires user input on launch cannot be used in saved launch configuration.")}
 
         ask_field_name = ask_mapping[self.relationship]
 
@@ -2501,7 +2501,7 @@ class JobTemplateSurveySpec(GenericAPIView):
                     return Response(
                         dict(
                             error=_(
-                                "$encrypted$ is a reserved keyword for password question defaults, " "survey question {idx} is type {survey_item[type]}."
+                                "$encrypted$ is a reserved keyword for password question defaults, survey question {idx} is type {survey_item[type]}."
                             ).format(**context)
                         ),
                         status=status.HTTP_400_BAD_REQUEST,
@@ -3333,7 +3333,6 @@ class JobLabelList(SubListAPIView):
     serializer_class = serializers.LabelSerializer
     parent_model = models.Job
     relationship = 'labels'
-    parent_key = 'job'
 
 
 class WorkflowJobLabelList(JobLabelList):
@@ -4056,7 +4055,7 @@ class UnifiedJobStdout(RetrieveAPIView):
                 return super(UnifiedJobStdout, self).retrieve(request, *args, **kwargs)
         except models.StdoutMaxBytesExceeded as e:
             response_message = _(
-                "Standard Output too large to display ({text_size} bytes), " "only download supported for sizes over {supported_size} bytes."
+                "Standard Output too large to display ({text_size} bytes), only download supported for sizes over {supported_size} bytes."
             ).format(text_size=e.total, supported_size=e.supported)
             if request.accepted_renderer.format == 'json':
                 return Response({'range': {'start': 0, 'end': 1, 'absolute_end': 1}, 'content': response_message})

awx/api/views/instance_install_bundle.py

@@ -57,13 +57,11 @@ class InstanceInstallBundle(GenericAPIView):
 
         with io.BytesIO() as f:
             with tarfile.open(fileobj=f, mode='w:gz') as tar:
-                # copy /etc/receptor/tls/ca/receptor-ca.crt to receptor/tls/ca in the tar file
-                tar.add(
-                    os.path.realpath('/etc/receptor/tls/ca/receptor-ca.crt'), arcname=f"{instance_obj.hostname}_install_bundle/receptor/tls/ca/receptor-ca.crt"
-                )
+                # copy /etc/receptor/tls/ca/mesh-CA.crt to receptor/tls/ca in the tar file
+                tar.add(os.path.realpath('/etc/receptor/tls/ca/mesh-CA.crt'), arcname=f"{instance_obj.hostname}_install_bundle/receptor/tls/ca/mesh-CA.crt")
 
-                # copy /etc/receptor/signing/work-public-key.pem to receptor/work-public-key.pem
-                tar.add('/etc/receptor/signing/work-public-key.pem', arcname=f"{instance_obj.hostname}_install_bundle/receptor/work-public-key.pem")
+                # copy /etc/receptor/work_public_key.pem to receptor/work_public_key.pem
+                tar.add('/etc/receptor/work_public_key.pem', arcname=f"{instance_obj.hostname}_install_bundle/receptor/work_public_key.pem")
 
                 # generate and write the receptor key to receptor/tls/receptor.key in the tar file
                 key, cert = generate_receptor_tls(instance_obj)
@@ -161,14 +159,14 @@ def generate_receptor_tls(instance_obj):
         .sign(key, hashes.SHA256())
     )
 
-    # sign csr with the receptor ca key from /etc/receptor/ca/receptor-ca.key
-    with open('/etc/receptor/tls/ca/receptor-ca.key', 'rb') as f:
+    # sign csr with the receptor ca key from /etc/receptor/ca/mesh-CA.key
+    with open('/etc/receptor/tls/ca/mesh-CA.key', 'rb') as f:
         ca_key = serialization.load_pem_private_key(
             f.read(),
             password=None,
         )
 
-    with open('/etc/receptor/tls/ca/receptor-ca.crt', 'rb') as f:
+    with open('/etc/receptor/tls/ca/mesh-CA.crt', 'rb') as f:
         ca_cert = x509.load_pem_x509_certificate(f.read())
 
     cert = (

awx/api/views/mixin.py

@@ -50,7 +50,7 @@ class UnifiedJobDeletionMixin(object):
                 return Response({"error": _("Job has not finished processing events.")}, status=status.HTTP_400_BAD_REQUEST)
             else:
                 # if it has been > 1 minute, events are probably lost
-                logger.warning('Allowing deletion of {} through the API without all events ' 'processed.'.format(obj.log_format))
+                logger.warning('Allowing deletion of {} through the API without all events processed.'.format(obj.log_format))
 
         # Manually cascade delete events if unpartitioned job
         if obj.has_unpartitioned_events:

awx/api/views/root.py

@@ -20,6 +20,7 @@ from rest_framework import status
 
 import requests
 
+from awx import MODE
 from awx.api.generics import APIView
 from awx.conf.registry import settings_registry
 from awx.main.analytics import all_collectors
@@ -54,6 +55,8 @@ class ApiRootView(APIView):
         data['custom_logo'] = settings.CUSTOM_LOGO
         data['custom_login_info'] = settings.CUSTOM_LOGIN_INFO
         data['login_redirect_override'] = settings.LOGIN_REDIRECT_OVERRIDE
+        if MODE == 'development':
+            data['swagger'] = drf_reverse('api:schema-swagger-ui')
         return Response(data)
 
 

awx/api/views/webhooks.py

@@ -114,7 +114,7 @@ class WebhookReceiverBase(APIView):
         # Ensure that the full contents of the request are captured for multiple uses.
         request.body
 
-        logger.debug("headers: {}\n" "data: {}\n".format(request.headers, request.data))
+        logger.debug("headers: {}\ndata: {}\n".format(request.headers, request.data))
         obj = self.get_object()
         self.check_signature(obj)
 

awx/conf/migrations/0010_change_to_JSONField.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2 on 2023-06-09 19:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('conf', '0009_rename_proot_settings'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='setting',
+            name='value',
+            field=models.JSONField(null=True),
+        ),
+    ]

awx/conf/models.py

@@ -8,7 +8,6 @@ import json
 from django.db import models
 
 # AWX
-from awx.main.fields import JSONBlob
 from awx.main.models.base import CreatedModifiedModel, prevent_search
 from awx.main.utils import encrypt_field
 from awx.conf import settings_registry
@@ -18,7 +17,7 @@ __all__ = ['Setting']
 
 class Setting(CreatedModifiedModel):
     key = models.CharField(max_length=255)
-    value = JSONBlob(null=True)
+    value = models.JSONField(null=True)
     user = prevent_search(models.ForeignKey('auth.User', related_name='settings', default=None, null=True, editable=False, on_delete=models.CASCADE))
 
     def __str__(self):

awx/conf/tests/unit/test_fields.py

@@ -35,7 +35,7 @@ class TestStringListBooleanField:
         field = StringListBooleanField()
         with pytest.raises(ValidationError) as e:
             field.to_internal_value(value)
-        assert e.value.detail[0] == "Expected None, True, False, a string or list " "of strings but got {} instead.".format(type(value))
+        assert e.value.detail[0] == "Expected None, True, False, a string or list of strings but got {} instead.".format(type(value))
 
     @pytest.mark.parametrize("value_in, value_known", FIELD_VALUES)
     def test_to_representation_valid(self, value_in, value_known):
@@ -48,7 +48,7 @@ class TestStringListBooleanField:
         field = StringListBooleanField()
         with pytest.raises(ValidationError) as e:
             field.to_representation(value)
-        assert e.value.detail[0] == "Expected None, True, False, a string or list " "of strings but got {} instead.".format(type(value))
+        assert e.value.detail[0] == "Expected None, True, False, a string or list of strings but got {} instead.".format(type(value))
 
 
 class TestListTuplesField:
@@ -67,7 +67,7 @@ class TestListTuplesField:
         field = ListTuplesField()
         with pytest.raises(ValidationError) as e:
             field.to_internal_value(value)
-        assert e.value.detail[0] == "Expected a list of tuples of max length 2 " "but got {} instead.".format(t)
+        assert e.value.detail[0] == "Expected a list of tuples of max length 2 but got {} instead.".format(t)
 
 
 class TestStringListPathField:

awx/main/access.py

@@ -2234,7 +2234,7 @@ class WorkflowJobAccess(BaseAccess):
             if not node_access.can_add({'reference_obj': node}):
                 wj_add_perm = False
         if not wj_add_perm and self.save_messages:
-            self.messages['workflow_job_template'] = _('You do not have permission to the workflow job ' 'resources required for relaunch.')
+            self.messages['workflow_job_template'] = _('You do not have permission to the workflow job resources required for relaunch.')
         return wj_add_perm
 
     def can_cancel(self, obj):

awx/main/analytics/collectors.py

@@ -399,7 +399,10 @@ def _copy_table(table, query, path):
     file_path = os.path.join(path, table + '_table.csv')
     file = FileSplitter(filespec=file_path)
     with connection.cursor() as cursor:
-        cursor.copy_expert(query, file)
+        with cursor.copy(query) as copy:
+            while data := copy.read():
+                byte_data = bytes(data)
+                file.write(byte_data.decode())
     return file.file_list()
 
 

awx/main/analytics/subsystem_metrics.py

@@ -209,6 +209,11 @@ class Metrics:
             SetFloatM('workflow_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
             SetFloatM('workflow_manager_spawn_workflow_graph_jobs_seconds', 'Time spent spawning workflow tasks'),
             SetFloatM('workflow_manager_get_tasks_seconds', 'Time spent loading workflow tasks from db'),
+            # dispatcher subsystem metrics
+            SetIntM('dispatcher_pool_scale_up_events', 'Number of times local dispatcher scaled up a worker since startup'),
+            SetIntM('dispatcher_pool_active_task_count', 'Number of active tasks in the worker pool when last task was submitted'),
+            SetIntM('dispatcher_pool_max_worker_count', 'Highest number of workers in worker pool in last collection interval, about 20s'),
+            SetFloatM('dispatcher_availability', 'Fraction of time (in last collection interval) dispatcher was able to receive messages'),
         ]
         # turn metric list into dictionary with the metric name as a key
         self.METRICS = {}

awx/main/conf.py

@@ -684,11 +684,28 @@ register(
     field_class=fields.IntegerField,
     default=1,
     min_value=1,
-    label=_('Maximum disk persistance for external log aggregation (in GB)'),
+    label=_('Maximum disk persistence for external log aggregation (in GB)'),
     help_text=_(
         'Amount of data to store (in gigabytes) during an outage of '
         'the external log aggregator (defaults to 1). '
-        'Equivalent to the rsyslogd queue.maxdiskspace setting.'
+        'Equivalent to the rsyslogd queue.maxdiskspace setting for main_queue. '
+        'Notably, this is used for the rsyslogd main queue (for input messages).'
+    ),
+    category=_('Logging'),
+    category_slug='logging',
+)
+register(
+    'LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB',
+    field_class=fields.IntegerField,
+    default=1,
+    min_value=1,
+    label=_('Maximum disk persistence for rsyslogd action queuing (in GB)'),
+    help_text=_(
+        'Amount of data to store (in gigabytes) if an rsyslog action takes time '
+        'to process an incoming message (defaults to 1). '
+        'Equivalent to the rsyslogd queue.maxdiskspace setting on the action (e.g. omhttp). '
+        'Like LOG_AGGREGATOR_MAX_DISK_USAGE_GB, it stores files in the directory specified '
+        'by LOG_AGGREGATOR_MAX_DISK_USAGE_PATH.'
     ),
     category=_('Logging'),
     category_slug='logging',
@@ -831,6 +848,46 @@ register(
     category_slug='system',
 )
 
+register(
+    'AWX_CLEANUP_PATHS',
+    field_class=fields.BooleanField,
+    label=_('Enable or Disable tmp dir cleanup'),
+    default=True,
+    help_text=_('Enable or Disable TMP Dir cleanup'),
+    category=('Debug'),
+    category_slug='debug',
+)
+
+register(
+    'AWX_REQUEST_PROFILE',
+    field_class=fields.BooleanField,
+    label=_('Debug Web Requests'),
+    default=False,
+    help_text=_('Debug web request python timing'),
+    category=('Debug'),
+    category_slug='debug',
+)
+
+register(
+    'DEFAULT_CONTAINER_RUN_OPTIONS',
+    field_class=fields.StringListField,
+    label=_('Container Run Options'),
+    default=['--network', 'slirp4netns:enable_ipv6=true'],
+    help_text=_("List of options to pass to podman run example: ['--network', 'slirp4netns:enable_ipv6=true', '--log-level', 'debug']"),
+    category=('Jobs'),
+    category_slug='jobs',
+)
+
+register(
+    'RECEPTOR_RELEASE_WORK',
+    field_class=fields.BooleanField,
+    label=_('Release Receptor Work'),
+    default=True,
+    help_text=_('Release receptor work'),
+    category=('Debug'),
+    category_slug='debug',
+)
+
 
 def logging_validate(serializer, attrs):
     if not serializer.instance or not hasattr(serializer.instance, 'LOG_AGGREGATOR_HOST') or not hasattr(serializer.instance, 'LOG_AGGREGATOR_TYPE'):

awx/main/credential_plugins/aws_secretsmanager.py

@@ -0,0 +1,65 @@
+import boto3
+from botocore.exceptions import ClientError
+
+from .plugin import CredentialPlugin
+from django.utils.translation import gettext_lazy as _
+
+
+secrets_manager_inputs = {
+    'fields': [
+        {
+            'id': 'aws_access_key',
+            'label': _('AWS Access Key'),
+            'type': 'string',
+        },
+        {
+            'id': 'aws_secret_key',
+            'label': _('AWS Secret Key'),
+            'type': 'string',
+            'secret': True,
+        },
+    ],
+    'metadata': [
+        {
+            'id': 'region_name',
+            'label': _('AWS Secrets Manager Region'),
+            'type': 'string',
+            'help_text': _('Region which the secrets manager is located'),
+        },
+        {
+            'id': 'secret_name',
+            'label': _('AWS Secret Name'),
+            'type': 'string',
+        },
+    ],
+    'required': ['aws_access_key', 'aws_secret_key', 'region_name', 'secret_name'],
+}
+
+
+def aws_secretsmanager_backend(**kwargs):
+    secret_name = kwargs['secret_name']
+    region_name = kwargs['region_name']
+    aws_secret_access_key = kwargs['aws_secret_key']
+    aws_access_key_id = kwargs['aws_access_key']
+
+    session = boto3.session.Session()
+    client = session.client(
+        service_name='secretsmanager', region_name=region_name, aws_secret_access_key=aws_secret_access_key, aws_access_key_id=aws_access_key_id
+    )
+
+    try:
+        get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+    except ClientError as e:
+        raise e
+    # Secrets Manager decrypts the secret value using the associated KMS CMK
+    # Depending on whether the secret was a string or binary, only one of these fields will be populated
+    if 'SecretString' in get_secret_value_response:
+        secret = get_secret_value_response['SecretString']
+
+    else:
+        secret = get_secret_value_response['SecretBinary']
+
+    return secret
+
+
+aws_secretmanager_plugin = CredentialPlugin('AWS Secrets Manager lookup', inputs=secrets_manager_inputs, backend=aws_secretsmanager_backend)

awx/main/credential_plugins/tss.py

@@ -50,7 +50,7 @@ tss_inputs = {
 
 
 def tss_backend(**kwargs):
-    if 'domain' in kwargs:
+    if kwargs.get("domain"):
         authorizer = DomainPasswordGrantAuthorizer(kwargs['server_url'], kwargs['username'], kwargs['password'], kwargs['domain'])
     else:
         authorizer = PasswordGrantAuthorizer(kwargs['server_url'], kwargs['username'], kwargs['password'])

awx/main/db/profiled_pg/base.py

@@ -87,7 +87,7 @@ class RecordedQueryLog(object):
             )
             log.commit()
             log.execute(
-                'INSERT INTO queries (pid, version, argv, time, sql, explain, bt) ' 'VALUES (?, ?, ?, ?, ?, ?, ?);',
+                'INSERT INTO queries (pid, version, argv, time, sql, explain, bt) VALUES (?, ?, ?, ?, ?, ?, ?);',
                 (os.getpid(), version, ' '.join(sys.argv), seconds, sql, explain, bt),
             )
             log.commit()

awx/main/dispatch/__init__.py

@@ -1,5 +1,5 @@
 import os
-import psycopg2
+import psycopg
 import select
 
 from contextlib import contextmanager
@@ -64,9 +64,9 @@ class PubSub(object):
                 if yield_timeouts:
                     yield None
             else:
-                self.conn.poll()
-                while self.conn.notifies:
-                    yield self.conn.notifies.pop(0)
+                notification_generator = self.conn.notifies()
+                for notification in notification_generator:
+                    yield notification
 
     def close(self):
         self.conn.close()
@@ -89,9 +89,8 @@ def pg_bus_conn(new_connection=False):
         conf['OPTIONS'] = conf.get('OPTIONS', {}).copy()
         # Modify the application name to distinguish from other connections the process might use
         conf['OPTIONS']['application_name'] = get_application_name(settings.CLUSTER_HOST_ID, function='listener')
-        conn = psycopg2.connect(dbname=conf['NAME'], host=conf['HOST'], user=conf['USER'], password=conf['PASSWORD'], port=conf['PORT'], **conf['OPTIONS'])
-        # Django connection.cursor().connection doesn't have autocommit=True on by default
-        conn.set_session(autocommit=True)
+        connection_data = f"dbname={conf['NAME']} host={conf['HOST']} user={conf['USER']} password={conf['PASSWORD']} port={conf['PORT']}"
+        conn = psycopg.connect(connection_data, autocommit=True, **conf['OPTIONS'])
     else:
         if pg_connection.connection is None:
             pg_connection.connect()

awx/main/dispatch/pool.py

@@ -339,6 +339,17 @@ class AutoscalePool(WorkerPool):
         # but if the task takes longer than the time defined here, we will force it to stop here
         self.task_manager_timeout = settings.TASK_MANAGER_TIMEOUT + settings.TASK_MANAGER_TIMEOUT_GRACE_PERIOD
 
+        # initialize some things for subsystem metrics periodic gathering
+        # the AutoscalePool class does not save these to redis directly, but reports via produce_subsystem_metrics
+        self.scale_up_ct = 0
+        self.worker_count_max = 0
+
+    def produce_subsystem_metrics(self, metrics_object):
+        metrics_object.set('dispatcher_pool_scale_up_events', self.scale_up_ct)
+        metrics_object.set('dispatcher_pool_active_task_count', sum(len(w.managed_tasks) for w in self.workers))
+        metrics_object.set('dispatcher_pool_max_worker_count', self.worker_count_max)
+        self.worker_count_max = len(self.workers)
+
     @property
     def should_grow(self):
         if len(self.workers) < self.min_workers:
@@ -443,7 +454,12 @@ class AutoscalePool(WorkerPool):
             idx = random.choice(range(len(self.workers)))
             return idx, self.workers[idx]
         else:
-            return super(AutoscalePool, self).up()
+            self.scale_up_ct += 1
+            ret = super(AutoscalePool, self).up()
+            new_worker_ct = len(self.workers)
+            if new_worker_ct > self.worker_count_max:
+                self.worker_count_max = new_worker_ct
+            return ret
 
     def write(self, preferred_queue, body):
         if 'guid' in body:

awx/main/dispatch/worker/base.py

@@ -7,7 +7,7 @@ import signal
 import sys
 import redis
 import json
-import psycopg2
+import psycopg
 import time
 from uuid import UUID
 from queue import Empty as QueueEmpty
@@ -19,6 +19,7 @@ from awx.main.dispatch.pool import WorkerPool
 from awx.main.dispatch import pg_bus_conn
 from awx.main.utils.common import log_excess_runtime
 from awx.main.utils.db import set_connection_name
+import awx.main.analytics.subsystem_metrics as s_metrics
 
 if 'run_callback_receiver' in sys.argv:
     logger = logging.getLogger('awx.main.commands.run_callback_receiver')
@@ -142,9 +143,10 @@ class AWXConsumerRedis(AWXConsumerBase):
     def run(self, *args, **kwargs):
         super(AWXConsumerRedis, self).run(*args, **kwargs)
         self.worker.on_start()
+        logger.info(f'Callback receiver started with pid={os.getpid()}')
+        db.connection.close()  # logs use database, so close connection
 
         while True:
-            logger.debug(f'{os.getpid()} is alive')
             time.sleep(60)
 
 
@@ -154,17 +156,30 @@ class AWXConsumerPG(AWXConsumerBase):
         self.pg_max_wait = settings.DISPATCHER_DB_DOWNTOWN_TOLLERANCE
         # if no successful loops have ran since startup, then we should fail right away
         self.pg_is_down = True  # set so that we fail if we get database errors on startup
-        self.pg_down_time = time.time() - self.pg_max_wait  # allow no grace period
-        self.last_cleanup = time.time()
+        init_time = time.time()
+        self.pg_down_time = init_time - self.pg_max_wait  # allow no grace period
+        self.last_cleanup = init_time
+        self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False)
+        self.last_metrics_gather = init_time
+        self.listen_cumulative_time = 0.0
 
     def run_periodic_tasks(self):
         self.record_statistics()  # maintains time buffer in method
 
-        if time.time() - self.last_cleanup > 60:  # same as cluster_node_heartbeat
+        current_time = time.time()
+        if current_time - self.last_cleanup > 60:  # same as cluster_node_heartbeat
             # NOTE: if we run out of database connections, it is important to still run cleanup
             # so that we scale down workers and free up connections
             self.pool.cleanup()
-            self.last_cleanup = time.time()
+            self.last_cleanup = current_time
+
+        # record subsystem metrics for the dispatcher
+        if current_time - self.last_metrics_gather > 20:
+            self.pool.produce_subsystem_metrics(self.subsystem_metrics)
+            self.subsystem_metrics.set('dispatcher_availability', self.listen_cumulative_time / (current_time - self.last_metrics_gather))
+            self.subsystem_metrics.pipe_execute()
+            self.listen_cumulative_time = 0.0
+            self.last_metrics_gather = current_time
 
     def run(self, *args, **kwargs):
         super(AWXConsumerPG, self).run(*args, **kwargs)
@@ -180,17 +195,20 @@ class AWXConsumerPG(AWXConsumerBase):
                     if init is False:
                         self.worker.on_start()
                         init = True
+                    self.listen_start = time.time()
                     for e in conn.events(yield_timeouts=True):
+                        self.listen_cumulative_time += time.time() - self.listen_start
                         if e is not None:
                             self.process_task(json.loads(e.payload))
                         self.run_periodic_tasks()
                         self.pg_is_down = False
+                        self.listen_start = time.time()
                     if self.should_stop:
                         return
-            except psycopg2.InterfaceError:
+            except psycopg.InterfaceError:
                 logger.warning("Stale Postgres message bus connection, reconnecting")
                 continue
-            except (db.DatabaseError, psycopg2.OperationalError):
+            except (db.DatabaseError, psycopg.OperationalError):
                 # If we have attained stady state operation, tolerate short-term database hickups
                 if not self.pg_is_down:
                     logger.exception(f"Error consuming new events from postgres, will retry for {self.pg_max_wait} s")

awx/main/dispatch/worker/callback.py

@@ -191,7 +191,9 @@ class CallbackBrokerWorker(BaseWorker):
                             e._retry_count = retry_count
 
                             # special sanitization logic for postgres treatment of NUL 0x00 char
-                            if (retry_count == 1) and isinstance(exc_indv, ValueError) and ("\x00" in e.stdout):
+                            # This used to check the class of the exception but on the postgres3 upgrade it could appear
+                            #   as either DataError or ValueError, so now lets just try if its there.
+                            if (retry_count == 1) and ("\x00" in e.stdout):
                                 e.stdout = e.stdout.replace("\x00", "")
 
                             if retry_count >= self.INDIVIDUAL_EVENT_RETRIES:

awx/main/fields.py

@@ -67,10 +67,60 @@ def __enum_validate__(validator, enums, instance, schema):
 Draft4Validator.VALIDATORS['enum'] = __enum_validate__
 
 
+import logging
+
+logger = logging.getLogger('awx.main.fields')
+
+
 class JSONBlob(JSONField):
+    # Cringe... a JSONField that is back ended with a TextField.
+    # This field was a legacy custom field type that tl;dr; was a TextField
+    # Over the years, with Django upgrades, we were able to go to a JSONField instead of the custom field
+    # However, we didn't want to have large customers with millions of events to update from text to json during an upgrade
+    # So we keep this field type as backended with TextField.
     def get_internal_type(self):
         return "TextField"
 
+    # postgres uses a Jsonb field as the default backend
+    # with psycopg2 it was using a psycopg2._json.Json class internally
+    # with psycopg3 it uses a psycopg.types.json.Jsonb class internally
+    # The binary class was not compatible with a text field, so we are going to override these next two methods and ensure we are using a string
+
+    def from_db_value(self, value, expression, connection):
+        if value is None:
+            return value
+
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except Exception as e:
+                logger.error(f"Failed to load JSONField {self.name}: {e}")
+
+        return value
+
+    def get_db_prep_value(self, value, connection, prepared=False):
+        if not prepared:
+            value = self.get_prep_value(value)
+        try:
+            # Null characters are not allowed in text fields and JSONBlobs are JSON data but saved as text
+            # So we want to make sure we strip out any null characters also note, these "should" be escaped by the dumps process:
+            #     >>> my_obj = { 'test': '\x00' }
+            #     >>> import json
+            #     >>> json.dumps(my_obj)
+            #     '{"test": "\\u0000"}'
+            # But just to be safe, lets remove them if they are there. \x00 and \u0000 are the same:
+            #     >>> string = "\x00"
+            #     >>> "\u0000" in string
+            #     True
+            dumped_value = json.dumps(value)
+            if "\x00" in dumped_value:
+                dumped_value = dumped_value.replace("\x00", '')
+            return dumped_value
+        except Exception as e:
+            logger.error(f"Failed to dump JSONField {self.name}: {e} value: {value}")
+
+        return value
+
 
 # Based on AutoOneToOneField from django-annoying:
 # https://bitbucket.org/offline/django-annoying/src/a0de8b294db3/annoying/fields.py
@@ -800,7 +850,7 @@ class CredentialTypeInjectorField(JSONSchemaField):
     def validate_env_var_allowed(self, env_var):
         if env_var.startswith('ANSIBLE_'):
             raise django_exceptions.ValidationError(
-                _('Environment variable {} may affect Ansible configuration so its ' 'use is not allowed in credentials.').format(env_var),
+                _('Environment variable {} may affect Ansible configuration so its use is not allowed in credentials.').format(env_var),
                 code='invalid',
                 params={'value': env_var},
             )

awx/main/management/commands/cleanup_activitystream.py

@@ -23,7 +23,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--days', dest='days', type=int, default=90, metavar='N', help='Remove activity stream events more than N days old')
-        parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False, help='Dry run mode (show items that would ' 'be removed)')
+        parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False, help='Dry run mode (show items that would be removed)')
 
     def init_logging(self):
         log_levels = dict(enumerate([logging.ERROR, logging.INFO, logging.DEBUG, 0]))

awx/main/management/commands/cleanup_jobs.py

@@ -17,10 +17,7 @@ from django.utils.timezone import now
 
 # AWX
 from awx.main.models import Job, AdHocCommand, ProjectUpdate, InventoryUpdate, SystemJob, WorkflowJob, Notification
-
-
-def unified_job_class_to_event_table_name(job_class):
-    return f'main_{job_class().event_class.__name__.lower()}'
+from awx.main.utils import unified_job_class_to_event_table_name
 
 
 def partition_table_name(job_class, dt):
@@ -152,7 +149,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--days', dest='days', type=int, default=90, metavar='N', help='Remove jobs/updates executed more than N days ago. Defaults to 90.')
-        parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False, help='Dry run mode (show items that would ' 'be removed)')
+        parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False, help='Dry run mode (show items that would be removed)')
         parser.add_argument('--jobs', dest='only_jobs', action='store_true', default=False, help='Remove jobs')
         parser.add_argument('--ad-hoc-commands', dest='only_ad_hoc_commands', action='store_true', default=False, help='Remove ad hoc commands')
         parser.add_argument('--project-updates', dest='only_project_updates', action='store_true', default=False, help='Remove project updates')

awx/main/management/commands/custom_venv_associations.py

@@ -44,7 +44,7 @@ class Command(BaseCommand):
                         '- To list all (now deprecated) custom virtual environments run:',
                         'awx-manage list_custom_venvs',
                         '',
-                        '- To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:',
+                        '- To export the contents of a (deprecated) virtual environment, run the following command while supplying the path as an argument:',
                         'awx-manage export_custom_venv /path/to/venv',
                         '',
                         '- Run these commands with `-q` to remove tool tips.',

awx/main/management/commands/deprovision_instance.py

@@ -13,7 +13,7 @@ class Command(BaseCommand):
     Deprovision a cluster node
     """
 
-    help = 'Remove instance from the database. ' 'Specify `--hostname` to use this command.'
+    help = 'Remove instance from the database. Specify `--hostname` to use this command.'
 
     def add_arguments(self, parser):
         parser.add_argument('--hostname', dest='hostname', type=str, help='Hostname used during provisioning')

awx/main/management/commands/list_custom_venvs.py

@@ -22,7 +22,7 @@ class Command(BaseCommand):
                     '# Discovered Virtual Environments:',
                     '\n'.join(venvs),
                     '',
-                    '- To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:',
+                    '- To export the contents of a (deprecated) virtual environment, run the following command while supplying the path as an argument:',
                     'awx-manage export_custom_venv /path/to/venv',
                     '',
                     '- To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:',

awx/main/management/commands/precreate_partitions.py

@@ -0,0 +1,27 @@
+from django.utils.timezone import now
+from django.core.management.base import BaseCommand, CommandParser
+from datetime import timedelta
+from awx.main.utils.common import create_partition, unified_job_class_to_event_table_name
+from awx.main.models import Job, SystemJob, ProjectUpdate, InventoryUpdate, AdHocCommand
+
+
+class Command(BaseCommand):
+    """Command used to precreate database partitions to avoid pg_dump locks"""
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument('--count', dest='count', action='store', help='The amount of hours of partitions to create', type=int, default=1)
+
+    def _create_partitioned_tables(self, count):
+        tables = list()
+        for model in (Job, SystemJob, ProjectUpdate, InventoryUpdate, AdHocCommand):
+            tables.append(unified_job_class_to_event_table_name(model))
+        start = now()
+        while count > 0:
+            for table in tables:
+                create_partition(table, start)
+                print(f'Created partitions for {table} {start}')
+            start = start + timedelta(hours=1)
+            count -= 1
+
+    def handle(self, **options):
+        self._create_partitioned_tables(count=options.get('count'))

awx/main/management/commands/provision_instance.py

@@ -35,7 +35,7 @@ class Command(BaseCommand):
 
             from awx.main.management.commands.register_queue import RegisterQueue
 
-            (changed, instance) = Instance.objects.register(ip_address=os.environ.get('MY_POD_IP'), node_type='control', uuid=settings.SYSTEM_UUID)
+            (changed, instance) = Instance.objects.register(ip_address=os.environ.get('MY_POD_IP'), node_type='control', node_uuid=settings.SYSTEM_UUID)
             RegisterQueue(settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME, 100, 0, [], is_container_group=False).register()
             RegisterQueue(
                 settings.DEFAULT_EXECUTION_QUEUE_NAME,
@@ -48,7 +48,7 @@ class Command(BaseCommand):
                 max_concurrent_jobs=settings.DEFAULT_EXECUTION_QUEUE_MAX_CONCURRENT_JOBS,
             ).register()
         else:
-            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, uuid=uuid)
+            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, node_uuid=uuid)
         if changed:
             print("Successfully registered instance {}".format(hostname))
         else:

awx/main/management/commands/run_cache_clear.py

@@ -2,6 +2,7 @@ import logging
 import json
 
 from django.core.management.base import BaseCommand
+
 from awx.main.dispatch import pg_bus_conn
 from awx.main.dispatch.worker.task import TaskWorker
 
@@ -18,7 +19,7 @@ class Command(BaseCommand):
 
     def handle(self, *arg, **options):
         try:
-            with pg_bus_conn(new_connection=True) as conn:
+            with pg_bus_conn() as conn:
                 conn.listen("tower_settings_change")
                 for e in conn.events(yield_timeouts=True):
                     if e is not None:

awx/main/management/commands/run_dispatcher.py

@@ -3,7 +3,6 @@
 import logging
 import yaml
 
-from django.conf import settings
 from django.core.cache import cache as django_cache
 from django.core.management.base import BaseCommand
 from django.db import connection as django_connection
@@ -17,10 +16,6 @@ from awx.main.dispatch import periodic
 logger = logging.getLogger('awx.main.dispatch')
 
 
-def construct_bcast_queue_name(common_name):
-    return common_name + '_' + settings.CLUSTER_HOST_ID
-
-
 class Command(BaseCommand):
     help = 'Launch the task dispatcher'
 

awx/main/management/commands/run_heartbeet.py

@@ -1,74 +0,0 @@
-import json
-import logging
-import os
-import time
-import signal
-import sys
-
-from django.core.management.base import BaseCommand
-from django.conf import settings
-
-from awx.main.dispatch import pg_bus_conn
-
-logger = logging.getLogger('awx.main.commands.run_heartbeet')
-
-
-class Command(BaseCommand):
-    help = 'Launch the web server beacon (heartbeet)'
-
-    def print_banner(self):
-        heartbeet = r"""
-   **********   **********
- ************* *************
-*****************************
- ***********HEART***********
-  *************************
-     *******************
-       *************** _._
-         *********** /`._ `'.      __
-           *******   \  .\| \   _'`  `)
-             ***  (``_)  \| ).'` /`- /
-              *   `\ `;\_ `\\//`-'` /
-                    \ `'.'.| /  __/`
-                     `'--v_|/`'`
-                         __||-._
-                      /'` `-``  `'\\
-                     /         .'` )
-                     \  BEET  '    )
-                      \.          /
-                        '.     /'`
-                           `) |
-                            //
-                            '(.
-                             `\`.
-                               ``"""
-        print(heartbeet)
-
-    def construct_payload(self, action='online'):
-        payload = {
-            'hostname': settings.CLUSTER_HOST_ID,
-            'ip': os.environ.get('MY_POD_IP'),
-            'action': action,
-        }
-        return json.dumps(payload)
-
-    def notify_listener_and_exit(self, *args):
-        with pg_bus_conn(new_connection=False) as conn:
-            conn.notify('web_heartbeet', self.construct_payload(action='offline'))
-        sys.exit(0)
-
-    def do_hearbeat_loop(self):
-        with pg_bus_conn(new_connection=True) as conn:
-            while True:
-                logger.debug('Sending heartbeat')
-                conn.notify('web_heartbeet', self.construct_payload())
-                time.sleep(settings.BROADCAST_WEBSOCKET_BEACON_FROM_WEB_RATE_SECONDS)
-
-    def handle(self, *arg, **options):
-        self.print_banner()
-        signal.signal(signal.SIGTERM, self.notify_listener_and_exit)
-        signal.signal(signal.SIGINT, self.notify_listener_and_exit)
-
-        # Note: We don't really try any reconnect logic to pg_notify here,
-        # just let supervisor restart if we fail.
-        self.do_hearbeat_loop()

awx/main/management/commands/run_rsyslog_configurer.py

@@ -22,7 +22,7 @@ class Command(BaseCommand):
 
     def handle(self, *arg, **options):
         try:
-            with pg_bus_conn(new_connection=True) as conn:
+            with pg_bus_conn() as conn:
                 conn.listen("rsyslog_configurer")
                 # reconfigure rsyslog on start up
                 reconfigure_rsyslog()

awx/main/management/commands/run_ws_heartbeat.py

@@ -0,0 +1,45 @@
+import json
+import logging
+import os
+import time
+import signal
+import sys
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from awx.main.dispatch import pg_bus_conn
+
+logger = logging.getLogger('awx.main.commands.run_ws_heartbeat')
+
+
+class Command(BaseCommand):
+    help = 'Launch the web server beacon (ws_heartbeat)'
+
+    def construct_payload(self, action='online'):
+        payload = {
+            'hostname': settings.CLUSTER_HOST_ID,
+            'ip': os.environ.get('MY_POD_IP'),
+            'action': action,
+        }
+        return json.dumps(payload)
+
+    def notify_listener_and_exit(self, *args):
+        with pg_bus_conn(new_connection=False) as conn:
+            conn.notify('web_ws_heartbeat', self.construct_payload(action='offline'))
+        sys.exit(0)
+
+    def do_heartbeat_loop(self):
+        while True:
+            with pg_bus_conn() as conn:
+                logger.debug('Sending heartbeat')
+                conn.notify('web_ws_heartbeat', self.construct_payload())
+            time.sleep(settings.BROADCAST_WEBSOCKET_BEACON_FROM_WEB_RATE_SECONDS)
+
+    def handle(self, *arg, **options):
+        signal.signal(signal.SIGTERM, self.notify_listener_and_exit)
+        signal.signal(signal.SIGINT, self.notify_listener_and_exit)
+
+        # Note: We don't really try any reconnect logic to pg_notify here,
+        # just let supervisor restart if we fail.
+        self.do_heartbeat_loop()

awx/main/managers.py

@@ -2,6 +2,7 @@
 # All Rights Reserved.
 
 import logging
+import uuid
 from django.db import models
 from django.conf import settings
 from django.db.models.functions import Lower
@@ -114,7 +115,7 @@ class InstanceManager(models.Manager):
             return node[0]
         raise RuntimeError("No instance found with the current cluster host id")
 
-    def register(self, uuid=None, hostname=None, ip_address=None, node_type='hybrid', defaults=None):
+    def register(self, node_uuid=None, hostname=None, ip_address=None, node_type='hybrid', defaults=None):
         if not hostname:
             hostname = settings.CLUSTER_HOST_ID
 
@@ -131,8 +132,8 @@ class InstanceManager(models.Manager):
                         logger.warning("IP address {0} conflict detected, ip address unset for host {1}.".format(ip_address, other_hostname))
 
             # Return existing instance that matches hostname or UUID (default to UUID)
-            if uuid is not None and uuid != UUID_DEFAULT and self.filter(uuid=uuid).exists():
-                instance = self.filter(uuid=uuid)
+            if node_uuid is not None and node_uuid != UUID_DEFAULT and self.filter(uuid=node_uuid).exists():
+                instance = self.filter(uuid=node_uuid)
             else:
                 # if instance was not retrieved by uuid and hostname was, use the hostname
                 instance = self.filter(hostname=hostname)
@@ -170,9 +171,7 @@ class InstanceManager(models.Manager):
             }
             if defaults is not None:
                 create_defaults.update(defaults)
-            uuid_option = {}
-            if uuid is not None:
-                uuid_option = {'uuid': uuid}
+            uuid_option = {'uuid': node_uuid if node_uuid is not None else uuid.uuid4()}
             if node_type == 'execution' and 'version' not in create_defaults:
                 create_defaults['version'] = RECEPTOR_PENDING
             instance = self.create(hostname=hostname, ip_address=ip_address, node_type=node_type, **create_defaults, **uuid_option)

awx/main/middleware.py

@@ -122,7 +122,7 @@ class URLModificationMiddleware(MiddlewareMixin):
             field_class=fields.DictField,
             read_only=True,
             label=_('Formats of all available named urls'),
-            help_text=_('Read-only list of key-value pairs that shows the standard format of all ' 'available named URLs.'),
+            help_text=_('Read-only list of key-value pairs that shows the standard format of all available named URLs.'),
             category=_('Named URL'),
             category_slug='named-url',
         )

awx/main/migrations/0006_v320_release.py

@@ -2,9 +2,6 @@
 # Python
 from __future__ import unicode_literals
 
-# Psycopg2
-from psycopg2.extensions import AsIs
-
 # Django
 from django.db import connection, migrations, models, OperationalError, ProgrammingError
 from django.conf import settings
@@ -136,8 +133,8 @@ class Migration(migrations.Migration):
             ),
         ),
         migrations.RunSQL(
-            [("CREATE INDEX host_ansible_facts_default_gin ON %s USING gin" "(ansible_facts jsonb_path_ops);", [AsIs(Host._meta.db_table)])],
-            [('DROP INDEX host_ansible_facts_default_gin;', None)],
+            sql="CREATE INDEX host_ansible_facts_default_gin ON {} USING gin(ansible_facts jsonb_path_ops);".format(Host._meta.db_table),
+            reverse_sql='DROP INDEX host_ansible_facts_default_gin;',
         ),
         # SCM file-based inventories
         migrations.AddField(

awx/main/migrations/0113_v370_event_bigint.py

@@ -12,22 +12,17 @@ def migrate_event_data(apps, schema_editor):
     # https://www.postgresql.org/docs/9.1/datatype-numeric.html)
     for tblname in ('main_jobevent', 'main_inventoryupdateevent', 'main_projectupdateevent', 'main_adhoccommandevent', 'main_systemjobevent'):
         with connection.cursor() as cursor:
-            # rename the current event table
-            cursor.execute(f'ALTER TABLE {tblname} RENAME TO _old_{tblname};')
-            # create a *new* table with the same schema
-            cursor.execute(f'CREATE TABLE {tblname} (LIKE _old_{tblname} INCLUDING ALL);')
-            # alter the *new* table so that the primary key is a big int
+            # This loop used to do roughly the following:
+            #     Rename the table to _old_<tablename>
+            #     Create a new table form the old table (it would have no rows)
+            #     Drop the old sequnce and create a new on tied to the new table and set the sequence to the last number from the old table
+            # This used to work with postgres spitting out a NOTICE and DETAIL
+            # With the django 4.2 upgrade that changed to an ERROR and HINT
+            # By the time we hit the 4.2 upgrade, no one should be upgrading a database this old directly to this new schema
+            # So we no longer really care about having to do all of this work, we only need a table with a bigint ID field
+            # And this can be achieved by just changing the id column type...
             cursor.execute(f'ALTER TABLE {tblname} ALTER COLUMN id TYPE bigint USING id::bigint;')
 
-            # recreate counter for the new table's primary key to
-            # start where the *old* table left off (we have to do this because the
-            # counter changed from an int to a bigint)
-            cursor.execute(f'DROP SEQUENCE IF EXISTS "{tblname}_id_seq" CASCADE;')
-            cursor.execute(f'CREATE SEQUENCE "{tblname}_id_seq";')
-            cursor.execute(f'ALTER TABLE "{tblname}" ALTER COLUMN "id" ' f"SET DEFAULT nextval('{tblname}_id_seq');")
-            cursor.execute(f"SELECT setval('{tblname}_id_seq', (SELECT MAX(id) FROM _old_{tblname}), true);")
-            cursor.execute(f'DROP TABLE _old_{tblname};')
-
 
 class FakeAlterField(migrations.AlterField):
     def database_forwards(self, *args):

awx/main/migrations/0144_event_partitions.py

@@ -30,7 +30,7 @@ def migrate_event_data(apps, schema_editor):
             # otherwise, the schema changes we would make on the old jobevents table
             # (namely, dropping the primary key constraint) would cause the migration
             # to suffer a serious performance degradation
-            cursor.execute(f'CREATE TABLE tmp_{tblname} ' f'(LIKE _unpartitioned_{tblname} INCLUDING ALL)')
+            cursor.execute(f'CREATE TABLE tmp_{tblname} (LIKE _unpartitioned_{tblname} INCLUDING ALL)')
 
             # drop primary key constraint; in a partioned table
             # constraints must include the partition key itself
@@ -48,7 +48,7 @@ def migrate_event_data(apps, schema_editor):
             cursor.execute(f'DROP TABLE tmp_{tblname}')
 
             # recreate primary key constraint
-            cursor.execute(f'ALTER TABLE ONLY {tblname} ' f'ADD CONSTRAINT {tblname}_pkey_new PRIMARY KEY (id, job_created);')
+            cursor.execute(f'ALTER TABLE ONLY {tblname} ADD CONSTRAINT {tblname}_pkey_new PRIMARY KEY (id, job_created);')
 
     with connection.cursor() as cursor:
         """

awx/main/migrations/0183_pre_django_upgrade.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.2.16 on 2023-04-21 14:15
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.OAUTH2_PROVIDER_ID_TOKEN_MODEL),
+        ('main', '0182_constructed_inventory'),
+        ('oauth2_provider', '0005_auto_20211222_2352'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='oauth2accesstoken',
+            name='id_token',
+            field=models.OneToOneField(
+                blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='access_token', to=settings.OAUTH2_PROVIDER_ID_TOKEN_MODEL
+            ),
+        ),
+        migrations.AddField(
+            model_name='oauth2application',
+            name='algorithm',
+            field=models.CharField(
+                blank=True, choices=[('', 'No OIDC support'), ('RS256', 'RSA with SHA-2 256'), ('HS256', 'HMAC with SHA-2 256')], default='', max_length=5
+            ),
+        ),
+    ]

awx/main/migrations/0185_move_JSONBlob_to_JSONField.py

@@ -0,0 +1,142 @@
+# Generated by Django 4.2 on 2023-06-09 19:51
+
+import awx.main.models.notifications
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0184_django_indexes'),
+        ('conf', '0010_change_to_JSONField'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='activitystream',
+            name='deleted_actor',
+            field=models.JSONField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='activitystream',
+            name='setting',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='instancegroup',
+            name='policy_instance_list',
+            field=models.JSONField(
+                blank=True, default=list, help_text='List of exact-match Instances that will always be automatically assigned to this group'
+            ),
+        ),
+        migrations.AlterField(
+            model_name='job',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='joblaunchconfig',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='joblaunchconfig',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='jobtemplate',
+            name='survey_spec',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='notification',
+            name='body',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='notificationtemplate',
+            name='messages',
+            field=models.JSONField(
+                blank=True,
+                default=awx.main.models.notifications.NotificationTemplate.default_messages,
+                help_text='Optional custom messages for notification template.',
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='notificationtemplate',
+            name='notification_configuration',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='inventory_files',
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                editable=False,
+                help_text='Suggested list of content that could be Ansible inventory in the project',
+                verbose_name='Inventory Files',
+            ),
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='playbook_files',
+            field=models.JSONField(blank=True, default=list, editable=False, help_text='List of playbooks found in the project', verbose_name='Playbook Files'),
+        ),
+        migrations.AlterField(
+            model_name='schedule',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='schedule',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='job_env',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='workflowjob',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='workflowjob',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobnode',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobnode',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplate',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplate',
+            name='survey_spec',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplatenode',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplatenode',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+    ]

awx/main/migrations/_OrgAdmin_to_use_ig.py

@@ -8,7 +8,7 @@ logger = logging.getLogger('awx.main.migrations')
 def migrate_org_admin_to_use(apps, schema_editor):
     logger.info('Initiated migration from Org admin to use role')
     roles_added = 0
-    for org in Organization.objects.prefetch_related('admin_role__members').iterator():
+    for org in Organization.objects.prefetch_related('admin_role__members').iterator(chunk_size=1000):
         igs = list(org.instance_groups.all())
         if not igs:
             continue

awx/main/migrations/_inventory_source_vars.py

@@ -158,7 +158,7 @@ class ec2(PluginFileInjector):
         return {
             # vars that change
             'ec2_block_devices': (
-                "dict(block_device_mappings | map(attribute='device_name') | list | zip(block_device_mappings " "| map(attribute='ebs.volume_id') | list))"
+                "dict(block_device_mappings | map(attribute='device_name') | list | zip(block_device_mappings | map(attribute='ebs.volume_id') | list))"
             ),
             'ec2_dns_name': 'public_dns_name',
             'ec2_group_name': 'placement.group_name',
@@ -635,7 +635,7 @@ class satellite6(PluginFileInjector):
             "environment": {
                 "prefix": "{}environment_".format(group_prefix),
                 "separator": "",
-                "key": "foreman['environment_name'] | lower | regex_replace(' ', '') | " "regex_replace('[^A-Za-z0-9_]', '_') | regex_replace('none', '')",
+                "key": "foreman['environment_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') | regex_replace('none', '')",
             },
             "location": {
                 "prefix": "{}location_".format(group_prefix),
@@ -656,7 +656,7 @@ class satellite6(PluginFileInjector):
             "content_view": {
                 "prefix": "{}content_view_".format(group_prefix),
                 "separator": "",
-                "key": "foreman['content_facet_attributes']['content_view_name'] | " "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')",
+                "key": "foreman['content_facet_attributes']['content_view_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')",
             },
         }
 

awx/main/models/activity_stream.py

@@ -3,7 +3,6 @@
 
 # AWX
 from awx.api.versioning import reverse
-from awx.main.fields import JSONBlob
 from awx.main.models.base import accepts_json
 
 # Django
@@ -36,7 +35,7 @@ class ActivityStream(models.Model):
     operation = models.CharField(max_length=13, choices=OPERATION_CHOICES)
     timestamp = models.DateTimeField(auto_now_add=True)
     changes = accepts_json(models.TextField(blank=True))
-    deleted_actor = JSONBlob(null=True)
+    deleted_actor = models.JSONField(null=True)
     action_node = models.CharField(
         blank=True,
         default='',
@@ -84,7 +83,7 @@ class ActivityStream(models.Model):
     o_auth2_application = models.ManyToManyField("OAuth2Application", blank=True)
     o_auth2_access_token = models.ManyToManyField("OAuth2AccessToken", blank=True)
 
-    setting = JSONBlob(default=dict, blank=True)
+    setting = models.JSONField(default=dict, blank=True)
 
     def __str__(self):
         operation = self.operation if 'operation' in self.__dict__ else '_delayed_'

awx/main/models/credential/__init__.py

@@ -91,7 +91,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
         related_name='credentials',
         null=False,
         on_delete=models.CASCADE,
-        help_text=_('Specify the type of credential you want to create. Refer ' 'to the documentation for details on each type.'),
+        help_text=_('Specify the type of credential you want to create. Refer to the documentation for details on each type.'),
     )
     managed = models.BooleanField(default=False, editable=False)
     organization = models.ForeignKey(
@@ -103,7 +103,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
         related_name='credentials',
     )
     inputs = CredentialInputField(
-        blank=True, default=dict, help_text=_('Enter inputs using either JSON or YAML syntax. ' 'Refer to the documentation for example syntax.')
+        blank=True, default=dict, help_text=_('Enter inputs using either JSON or YAML syntax. Refer to the documentation for example syntax.')
     )
     admin_role = ImplicitRoleField(
         parent_role=[
@@ -195,6 +195,9 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
 
     @cached_property
     def dynamic_input_fields(self):
+        # if the credential is not yet saved we can't access the input_sources
+        if not self.id:
+            return []
         return [obj.input_field_name for obj in self.input_sources.all()]
 
     def _password_field_allows_ask(self, field):
@@ -343,12 +346,12 @@ class CredentialType(CommonModelNameNotUnique):
     managed = models.BooleanField(default=False, editable=False)
     namespace = models.CharField(max_length=1024, null=True, default=None, editable=False)
     inputs = CredentialTypeInputField(
-        blank=True, default=dict, help_text=_('Enter inputs using either JSON or YAML syntax. ' 'Refer to the documentation for example syntax.')
+        blank=True, default=dict, help_text=_('Enter inputs using either JSON or YAML syntax. Refer to the documentation for example syntax.')
     )
     injectors = CredentialTypeInjectorField(
         blank=True,
         default=dict,
-        help_text=_('Enter injectors using either JSON or YAML syntax. ' 'Refer to the documentation for example syntax.'),
+        help_text=_('Enter injectors using either JSON or YAML syntax. Refer to the documentation for example syntax.'),
     )
 
     @classmethod
@@ -602,9 +605,7 @@ ManagedCredentialType(
                 'id': 'become_method',
                 'label': gettext_noop('Privilege Escalation Method'),
                 'type': 'string',
-                'help_text': gettext_noop(
-                    'Specify a method for "become" operations. This is ' 'equivalent to specifying the --become-method ' 'Ansible parameter.'
-                ),
+                'help_text': gettext_noop('Specify a method for "become" operations. This is equivalent to specifying the --become-method Ansible parameter.'),
             },
             {
                 'id': 'become_username',
@@ -746,7 +747,7 @@ ManagedCredentialType(
                 'id': 'host',
                 'label': gettext_noop('Host (Authentication URL)'),
                 'type': 'string',
-                'help_text': gettext_noop('The host to authenticate with.  For example, ' 'https://openstack.business.com/v2.0/'),
+                'help_text': gettext_noop('The host to authenticate with.  For example, https://openstack.business.com/v2.0/'),
             },
             {
                 'id': 'project',
@@ -797,7 +798,7 @@ ManagedCredentialType(
                 'id': 'host',
                 'label': gettext_noop('VCenter Host'),
                 'type': 'string',
-                'help_text': gettext_noop('Enter the hostname or IP address that corresponds ' 'to your VMware vCenter.'),
+                'help_text': gettext_noop('Enter the hostname or IP address that corresponds to your VMware vCenter.'),
             },
             {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
             {
@@ -822,7 +823,7 @@ ManagedCredentialType(
                 'id': 'host',
                 'label': gettext_noop('Satellite 6 URL'),
                 'type': 'string',
-                'help_text': gettext_noop('Enter the URL that corresponds to your Red Hat ' 'Satellite 6 server. For example, https://satellite.example.org'),
+                'help_text': gettext_noop('Enter the URL that corresponds to your Red Hat Satellite 6 server. For example, https://satellite.example.org'),
             },
             {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
             {
@@ -847,7 +848,7 @@ ManagedCredentialType(
                 'id': 'username',
                 'label': gettext_noop('Service Account Email Address'),
                 'type': 'string',
-                'help_text': gettext_noop('The email address assigned to the Google Compute ' 'Engine service account.'),
+                'help_text': gettext_noop('The email address assigned to the Google Compute Engine service account.'),
             },
             {
                 'id': 'project',
@@ -867,7 +868,7 @@ ManagedCredentialType(
                 'format': 'ssh_private_key',
                 'secret': True,
                 'multiline': True,
-                'help_text': gettext_noop('Paste the contents of the PEM file associated ' 'with the service account email.'),
+                'help_text': gettext_noop('Paste the contents of the PEM file associated with the service account email.'),
             },
         ],
         'required': ['username', 'ssh_key_data'],
@@ -885,7 +886,7 @@ ManagedCredentialType(
                 'id': 'subscription',
                 'label': gettext_noop('Subscription ID'),
                 'type': 'string',
-                'help_text': gettext_noop('Subscription ID is an Azure construct, which is ' 'mapped to a username.'),
+                'help_text': gettext_noop('Subscription ID is an Azure construct, which is mapped to a username.'),
             },
             {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
             {
@@ -906,7 +907,7 @@ ManagedCredentialType(
                 'id': 'cloud_environment',
                 'label': gettext_noop('Azure Cloud Environment'),
                 'type': 'string',
-                'help_text': gettext_noop('Environment variable AZURE_CLOUD_ENVIRONMENT when' ' using Azure GovCloud or Azure stack.'),
+                'help_text': gettext_noop('Environment variable AZURE_CLOUD_ENVIRONMENT when using Azure GovCloud or Azure stack.'),
             },
         ],
         'required': ['subscription'],
@@ -1037,7 +1038,7 @@ ManagedCredentialType(
                 'label': gettext_noop('Username'),
                 'type': 'string',
                 'help_text': gettext_noop(
-                    'Red Hat Ansible Automation Platform username id to authenticate as.' 'This should not be set if an OAuth token is being used.'
+                    'Red Hat Ansible Automation Platform username id to authenticate as.This should not be set if an OAuth token is being used.'
                 ),
             },
             {
@@ -1051,7 +1052,7 @@ ManagedCredentialType(
                 'label': gettext_noop('OAuth Token'),
                 'type': 'string',
                 'secret': True,
-                'help_text': gettext_noop('An OAuth token to use to authenticate with.' 'This should not be set if username/password are being used.'),
+                'help_text': gettext_noop('An OAuth token to use to authenticate with.This should not be set if username/password are being used.'),
             },
             {'id': 'verify_ssl', 'label': gettext_noop('Verify SSL'), 'type': 'boolean', 'secret': False},
         ],
@@ -1162,7 +1163,7 @@ ManagedCredentialType(
                 'id': 'auth_url',
                 'label': gettext_noop('Auth Server URL'),
                 'type': 'string',
-                'help_text': gettext_noop('The URL of a Keycloak server token_endpoint, if using ' 'SSO auth.'),
+                'help_text': gettext_noop('The URL of a Keycloak server token_endpoint, if using SSO auth.'),
             },
             {
                 'id': 'token',

awx/main/models/events.py

@@ -1,8 +1,10 @@
 # -*- coding: utf-8 -*-
 
 import datetime
+from datetime import timezone
 import logging
 from collections import defaultdict
+import time
 
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
@@ -10,7 +12,7 @@ from django.db import models, DatabaseError
 from django.db.models.functions import Cast
 from django.utils.dateparse import parse_datetime
 from django.utils.text import Truncator
-from django.utils.timezone import utc, now
+from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 from django.utils.encoding import force_str
 
@@ -382,8 +384,17 @@ class BasePlaybookEvent(CreatedModifiedModel):
                         .distinct()
                     )  # noqa
 
-                    job.get_event_queryset().filter(uuid__in=changed).update(changed=True)
-                    job.get_event_queryset().filter(uuid__in=failed).update(failed=True)
+                    # NOTE: we take a set of changed and failed parent uuids because the subquery
+                    # complicates the plan with large event tables causing very long query execution time
+                    changed_start = time.time()
+                    changed_res = job.get_event_queryset().filter(uuid__in=set(changed)).update(changed=True)
+                    failed_start = time.time()
+                    failed_res = job.get_event_queryset().filter(uuid__in=set(failed)).update(failed=True)
+                    logger.debug(
+                        f'Event propagation for job {job.id}: '
+                        f'marked {changed_res} as changed in {failed_start - changed_start:.4f}s, '
+                        f'{failed_res} as failed in {time.time() - failed_start:.4f}s'
+                    )
 
         for field in ('playbook', 'play', 'task', 'role'):
             value = force_str(event_data.get(field, '')).strip()
@@ -422,7 +433,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
             if not isinstance(kwargs['created'], datetime.datetime):
                 kwargs['created'] = parse_datetime(kwargs['created'])
             if not kwargs['created'].tzinfo:
-                kwargs['created'] = kwargs['created'].replace(tzinfo=utc)
+                kwargs['created'] = kwargs['created'].replace(tzinfo=timezone.utc)
         except (KeyError, ValueError):
             kwargs.pop('created', None)
 
@@ -432,7 +443,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
             if not isinstance(kwargs['job_created'], datetime.datetime):
                 kwargs['job_created'] = parse_datetime(kwargs['job_created'])
             if not kwargs['job_created'].tzinfo:
-                kwargs['job_created'] = kwargs['job_created'].replace(tzinfo=utc)
+                kwargs['job_created'] = kwargs['job_created'].replace(tzinfo=timezone.utc)
         except (KeyError, ValueError):
             kwargs.pop('job_created', None)
 
@@ -470,11 +481,11 @@ class JobEvent(BasePlaybookEvent):
     class Meta:
         app_label = 'main'
         ordering = ('pk',)
-        index_together = [
-            ('job', 'job_created', 'event'),
-            ('job', 'job_created', 'uuid'),
-            ('job', 'job_created', 'parent_uuid'),
-            ('job', 'job_created', 'counter'),
+        indexes = [
+            models.Index(fields=['job', 'job_created', 'event']),
+            models.Index(fields=['job', 'job_created', 'uuid']),
+            models.Index(fields=['job', 'job_created', 'parent_uuid']),
+            models.Index(fields=['job', 'job_created', 'counter']),
         ]
 
     id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')
@@ -632,10 +643,10 @@ class ProjectUpdateEvent(BasePlaybookEvent):
     class Meta:
         app_label = 'main'
         ordering = ('pk',)
-        index_together = [
-            ('project_update', 'job_created', 'event'),
-            ('project_update', 'job_created', 'uuid'),
-            ('project_update', 'job_created', 'counter'),
+        indexes = [
+            models.Index(fields=['project_update', 'job_created', 'event']),
+            models.Index(fields=['project_update', 'job_created', 'uuid']),
+            models.Index(fields=['project_update', 'job_created', 'counter']),
         ]
 
     id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')
@@ -734,7 +745,7 @@ class BaseCommandEvent(CreatedModifiedModel):
             if not isinstance(kwargs['created'], datetime.datetime):
                 kwargs['created'] = parse_datetime(kwargs['created'])
             if not kwargs['created'].tzinfo:
-                kwargs['created'] = kwargs['created'].replace(tzinfo=utc)
+                kwargs['created'] = kwargs['created'].replace(tzinfo=timezone.utc)
         except (KeyError, ValueError):
             kwargs.pop('created', None)
 
@@ -770,10 +781,10 @@ class AdHocCommandEvent(BaseCommandEvent):
     class Meta:
         app_label = 'main'
         ordering = ('-pk',)
-        index_together = [
-            ('ad_hoc_command', 'job_created', 'event'),
-            ('ad_hoc_command', 'job_created', 'uuid'),
-            ('ad_hoc_command', 'job_created', 'counter'),
+        indexes = [
+            models.Index(fields=['ad_hoc_command', 'job_created', 'event']),
+            models.Index(fields=['ad_hoc_command', 'job_created', 'uuid']),
+            models.Index(fields=['ad_hoc_command', 'job_created', 'counter']),
         ]
 
     EVENT_TYPES = [
@@ -875,9 +886,9 @@ class InventoryUpdateEvent(BaseCommandEvent):
     class Meta:
         app_label = 'main'
         ordering = ('-pk',)
-        index_together = [
-            ('inventory_update', 'job_created', 'uuid'),
-            ('inventory_update', 'job_created', 'counter'),
+        indexes = [
+            models.Index(fields=['inventory_update', 'job_created', 'uuid']),
+            models.Index(fields=['inventory_update', 'job_created', 'counter']),
         ]
 
     id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')
@@ -920,9 +931,9 @@ class SystemJobEvent(BaseCommandEvent):
     class Meta:
         app_label = 'main'
         ordering = ('-pk',)
-        index_together = [
-            ('system_job', 'job_created', 'uuid'),
-            ('system_job', 'job_created', 'counter'),
+        indexes = [
+            models.Index(fields=['system_job', 'job_created', 'uuid']),
+            models.Index(fields=['system_job', 'job_created', 'counter']),
         ]
 
     id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')

awx/main/models/ha.py

@@ -20,7 +20,7 @@ from solo.models import SingletonModel
 # AWX
 from awx import __version__ as awx_application_version
 from awx.api.versioning import reverse
-from awx.main.fields import JSONBlob, ImplicitRoleField
+from awx.main.fields import ImplicitRoleField
 from awx.main.managers import InstanceManager, UUID_DEFAULT
 from awx.main.constants import JOB_FOLDER_PREFIX
 from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
@@ -406,7 +406,7 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin, ResourceMi
     max_forks = models.IntegerField(default=0, help_text=_("Max forks to execute on this group. Zero means no limit."))
     policy_instance_percentage = models.IntegerField(default=0, help_text=_("Percentage of Instances to automatically assign to this group"))
     policy_instance_minimum = models.IntegerField(default=0, help_text=_("Static minimum number of Instances to automatically assign to this group"))
-    policy_instance_list = JSONBlob(
+    policy_instance_list = models.JSONField(
         default=list, blank=True, help_text=_("List of exact-match Instances that will always be automatically assigned to this group")
     )
 

awx/main/models/inventory.py

@@ -106,28 +106,28 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
     has_active_failures = models.BooleanField(
         default=False,
         editable=False,
-        help_text=_('This field is deprecated and will be removed in a future release. ' 'Flag indicating whether any hosts in this inventory have failed.'),
+        help_text=_('This field is deprecated and will be removed in a future release. Flag indicating whether any hosts in this inventory have failed.'),
     )
     total_hosts = models.PositiveIntegerField(
         default=0,
         editable=False,
-        help_text=_('This field is deprecated and will be removed in a future release. ' 'Total number of hosts in this inventory.'),
+        help_text=_('This field is deprecated and will be removed in a future release. Total number of hosts in this inventory.'),
     )
     hosts_with_active_failures = models.PositiveIntegerField(
         default=0,
         editable=False,
-        help_text=_('This field is deprecated and will be removed in a future release. ' 'Number of hosts in this inventory with active failures.'),
+        help_text=_('This field is deprecated and will be removed in a future release. Number of hosts in this inventory with active failures.'),
     )
     total_groups = models.PositiveIntegerField(
         default=0,
         editable=False,
-        help_text=_('This field is deprecated and will be removed in a future release. ' 'Total number of groups in this inventory.'),
+        help_text=_('This field is deprecated and will be removed in a future release. Total number of groups in this inventory.'),
     )
     has_inventory_sources = models.BooleanField(
         default=False,
         editable=False,
         help_text=_(
-            'This field is deprecated and will be removed in a future release. ' 'Flag indicating whether this inventory has any external inventory sources.'
+            'This field is deprecated and will be removed in a future release. Flag indicating whether this inventory has any external inventory sources.'
         ),
     )
     total_inventory_sources = models.PositiveIntegerField(
@@ -424,7 +424,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
             for t in tasks:
                 t.task_impact = t._get_task_impact()
             UnifiedJob.objects.bulk_update(tasks, ['task_impact'])
-        logger.debug("Finished updating inventory computed fields, pk={0}, in " "{1:.3f} seconds".format(self.pk, time.time() - start_time))
+        logger.debug("Finished updating inventory computed fields, pk={0}, in {1:.3f} seconds".format(self.pk, time.time() - start_time))
 
     def websocket_emit_status(self, status):
         connection.on_commit(
@@ -1055,16 +1055,16 @@ class InventorySourceOptions(BaseModel):
             # the actual inventory source being used (Amazon requires Amazon
             # credentials; Rackspace requires Rackspace credentials; etc...)
             if source.replace('ec2', 'aws') != cred.kind:
-                return _('Cloud-based inventory sources (such as %s) require ' 'credentials for the matching cloud service.') % source
+                return _('Cloud-based inventory sources (such as %s) require credentials for the matching cloud service.') % source
         # Allow an EC2 source to omit the credential.  If Tower is running on
         # an EC2 instance with an IAM Role assigned, boto will use credentials
         # from the instance metadata instead of those explicitly provided.
         elif source in CLOUD_PROVIDERS and source != 'ec2':
             return _('Credential is required for a cloud source.')
         elif source == 'custom' and cred and cred.credential_type.kind in ('scm', 'ssh', 'insights', 'vault'):
-            return _('Credentials of type machine, source control, insights and vault are ' 'disallowed for custom inventory sources.')
+            return _('Credentials of type machine, source control, insights and vault are disallowed for custom inventory sources.')
         elif source == 'scm' and cred and cred.credential_type.kind in ('insights', 'vault'):
-            return _('Credentials of type insights and vault are ' 'disallowed for scm inventory sources.')
+            return _('Credentials of type insights and vault are disallowed for scm inventory sources.')
         return None
 
     def get_cloud_credential(self):
@@ -1623,6 +1623,7 @@ class rhv(PluginFileInjector):
     collection = 'ovirt'
     downstream_namespace = 'redhat'
     downstream_collection = 'rhv'
+    use_fqcn = True
 
 
 class satellite6(PluginFileInjector):

awx/main/models/jobs.py

@@ -101,7 +101,7 @@ class JobOptions(BaseModel):
         max_length=1024,
         default='',
         blank=True,
-        help_text=_('Branch to use in job run. Project default used if blank. ' 'Only allowed if project allow_override field is set to true.'),
+        help_text=_('Branch to use in job run. Project default used if blank. Only allowed if project allow_override field is set to true.'),
     )
     forks = models.PositiveIntegerField(
         blank=True,
@@ -253,7 +253,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
     job_slice_count = models.PositiveIntegerField(
         blank=True,
         default=1,
-        help_text=_("The number of jobs to slice into at runtime. " "Will cause the Job Template to launch a workflow if value is greater than 1."),
+        help_text=_("The number of jobs to slice into at runtime. Will cause the Job Template to launch a workflow if value is greater than 1."),
     )
 
     admin_role = ImplicitRoleField(parent_role=['organization.job_template_admin_role'])
@@ -596,12 +596,12 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
     job_slice_number = models.PositiveIntegerField(
         blank=True,
         default=0,
-        help_text=_("If part of a sliced job, the ID of the inventory slice operated on. " "If not part of sliced job, parameter is not used."),
+        help_text=_("If part of a sliced job, the ID of the inventory slice operated on. If not part of sliced job, parameter is not used."),
     )
     job_slice_count = models.PositiveIntegerField(
         blank=True,
         default=1,
-        help_text=_("If ran as part of sliced jobs, the total number of slices. " "If 1, job is not part of a sliced job."),
+        help_text=_("If ran as part of sliced jobs, the total number of slices. If 1, job is not part of a sliced job."),
     )
 
     def _get_parent_field_name(self):
@@ -883,7 +883,7 @@ class LaunchTimeConfigBase(BaseModel):
     )
     # All standard fields are stored in this dictionary field
     # This is a solution to the nullable CharField problem, specific to prompting
-    char_prompts = JSONBlob(default=dict, blank=True)
+    char_prompts = models.JSONField(default=dict, blank=True)
 
     # Define fields that are not really fields, but alias to char_prompts lookups
     limit = NullablePromptPseudoField('limit')
@@ -960,7 +960,7 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
     # Special case prompting fields, even more special than the other ones
     extra_data = JSONBlob(default=dict, blank=True)
     survey_passwords = prevent_search(
-        JSONBlob(
+        models.JSONField(
             default=dict,
             editable=False,
             blank=True,

awx/main/models/mixins.py

@@ -24,7 +24,7 @@ from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_lice
 from awx.main.utils.execution_environments import get_default_execution_environment
 from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
 from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
-from awx.main.fields import AskForField, JSONBlob
+from awx.main.fields import AskForField
 from awx.main.constants import ACTIVE_STATES
 
 
@@ -103,7 +103,7 @@ class SurveyJobTemplateMixin(models.Model):
     survey_enabled = models.BooleanField(
         default=False,
     )
-    survey_spec = prevent_search(JSONBlob(default=dict, blank=True))
+    survey_spec = prevent_search(models.JSONField(default=dict, blank=True))
 
     ask_inventory_on_launch = AskForField(
         blank=True,
@@ -392,7 +392,7 @@ class SurveyJobMixin(models.Model):
         abstract = True
 
     survey_passwords = prevent_search(
-        JSONBlob(
+        models.JSONField(
             default=dict,
             editable=False,
             blank=True,
@@ -675,4 +675,4 @@ class WebhookMixin(models.Model):
         if response.status_code < 400:
             logger.debug("Webhook status update sent.")
         else:
-            logger.error("Posting webhook status failed, code: {}\n" "{}\n" "Payload sent: {}".format(response.status_code, response.text, json.dumps(data)))
+            logger.error("Posting webhook status failed, code: {}\n" "{}\nPayload sent: {}".format(response.status_code, response.text, json.dumps(data)))

awx/main/models/notifications.py

@@ -17,7 +17,6 @@ from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
 
 # AWX
 from awx.api.versioning import reverse
-from awx.main.fields import JSONBlob
 from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search
 from awx.main.utils import encrypt_field, decrypt_field, set_environ
 from awx.main.notifications.email_backend import CustomEmailBackend
@@ -69,12 +68,12 @@ class NotificationTemplate(CommonModelNameNotUnique):
         choices=NOTIFICATION_TYPE_CHOICES,
     )
 
-    notification_configuration = prevent_search(JSONBlob(default=dict))
+    notification_configuration = prevent_search(models.JSONField(default=dict))
 
     def default_messages():
         return {'started': None, 'success': None, 'error': None, 'workflow_approval': None}
 
-    messages = JSONBlob(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.'))
+    messages = models.JSONField(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.'))
 
     def has_message(self, condition):
         potential_template = self.messages.get(condition, {})
@@ -236,7 +235,7 @@ class Notification(CreatedModifiedModel):
         default='',
         editable=False,
     )
-    body = JSONBlob(default=dict, blank=True)
+    body = models.JSONField(default=dict, blank=True)
 
     def get_absolute_url(self, request=None):
         return reverse('api:notification_detail', kwargs={'pk': self.pk}, request=request)

awx/main/models/projects.py

@@ -33,7 +33,7 @@ from awx.main.models.mixins import ResourceMixin, TaskManagerProjectUpdateMixin,
 from awx.main.utils import update_scm_url, polymorphic
 from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook
 from awx.main.utils.execution_environments import get_control_plane_execution_environment
-from awx.main.fields import ImplicitRoleField, JSONBlob
+from awx.main.fields import ImplicitRoleField
 from awx.main.models.rbac import (
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
     ROLE_SINGLETON_SYSTEM_AUDITOR,
@@ -74,7 +74,7 @@ class ProjectOptions(models.Model):
             return []
 
     local_path = models.CharField(
-        max_length=1024, blank=True, help_text=_('Local path (relative to PROJECTS_ROOT) containing ' 'playbooks and related files for this project.')
+        max_length=1024, blank=True, help_text=_('Local path (relative to PROJECTS_ROOT) containing playbooks and related files for this project.')
     )
 
     scm_type = models.CharField(
@@ -276,11 +276,11 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     scm_update_cache_timeout = models.PositiveIntegerField(
         default=0,
         blank=True,
-        help_text=_('The number of seconds after the last project update ran that a new ' 'project update will be launched as a job dependency.'),
+        help_text=_('The number of seconds after the last project update ran that a new project update will be launched as a job dependency.'),
     )
     allow_override = models.BooleanField(
         default=False,
-        help_text=_('Allow changing the SCM branch or revision in a job template ' 'that uses this project.'),
+        help_text=_('Allow changing the SCM branch or revision in a job template that uses this project.'),
     )
 
     # credential (keys) used to validate content signature
@@ -303,7 +303,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         help_text=_('The last revision fetched by a project update'),
     )
 
-    playbook_files = JSONBlob(
+    playbook_files = models.JSONField(
         default=list,
         blank=True,
         editable=False,
@@ -311,7 +311,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         help_text=_('List of playbooks found in the project'),
     )
 
-    inventory_files = JSONBlob(
+    inventory_files = models.JSONField(
         default=list,
         blank=True,
         editable=False,
@@ -479,7 +479,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         RunProjectUpdate/RunInventoryUpdate.
         """
 
-        if self.status not in ('error', 'failed'):
+        if self.status not in ('error', 'failed') or self.scm_update_on_launch:
             return None
 
         latest_update = self.project_updates.last()

awx/main/models/rbac.py

@@ -141,7 +141,7 @@ class Role(models.Model):
         app_label = 'main'
         verbose_name_plural = _('roles')
         db_table = 'main_rbac_roles'
-        index_together = [("content_type", "object_id")]
+        indexes = [models.Index(fields=["content_type", "object_id"])]
         ordering = ("content_type", "object_id")
 
     role_field = models.TextField(null=False)
@@ -447,10 +447,10 @@ class RoleAncestorEntry(models.Model):
         app_label = 'main'
         verbose_name_plural = _('role_ancestors')
         db_table = 'main_rbac_role_ancestors'
-        index_together = [
-            ("ancestor", "content_type_id", "object_id"),  # used by get_roles_on_resource
-            ("ancestor", "content_type_id", "role_field"),  # used by accessible_objects
-            ("ancestor", "descendent"),  # used by rebuild_role_ancestor_list in the NOT EXISTS clauses.
+        indexes = [
+            models.Index(fields=["ancestor", "content_type_id", "object_id"]),  # used by get_roles_on_resource
+            models.Index(fields=["ancestor", "content_type_id", "role_field"]),  # used by accessible_objects
+            models.Index(fields=["ancestor", "descendent"]),  # used by rebuild_role_ancestor_list in the NOT EXISTS clauses.
         ]
 
     descendent = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')

awx/main/models/unified_jobs.py

@@ -55,7 +55,7 @@ from awx.main.utils import polymorphic
 from awx.main.constants import ACTIVE_STATES, CAN_CANCEL, JOB_VARIABLE_PREFIXES
 from awx.main.redact import UriCleaner, REPLACE_STR
 from awx.main.consumers import emit_channel_notification
-from awx.main.fields import AskForField, OrderedManyToManyField, JSONBlob
+from awx.main.fields import AskForField, OrderedManyToManyField
 
 __all__ = ['UnifiedJobTemplate', 'UnifiedJob', 'StdoutMaxBytesExceeded']
 
@@ -668,7 +668,7 @@ class UnifiedJob(
         editable=False,
     )
     job_env = prevent_search(
-        JSONBlob(
+        models.JSONField(
             default=dict,
             blank=True,
             editable=False,
@@ -1137,11 +1137,6 @@ class UnifiedJob(
                     if total > max_supported:
                         raise StdoutMaxBytesExceeded(total, max_supported)
 
-                # psycopg2's copy_expert writes bytes, but callers of this
-                # function assume a str-based fd will be returned; decode
-                # .write() calls on the fly to maintain this interface
-                _write = fd.write
-                fd.write = lambda s: _write(smart_str(s))
                 tbl = self._meta.db_table + 'event'
                 created_by_cond = ''
                 if self.has_unpartitioned_events:
@@ -1150,7 +1145,12 @@ class UnifiedJob(
                     created_by_cond = f"job_created='{self.created.isoformat()}' AND "
 
                 sql = f"copy (select stdout from {tbl} where {created_by_cond}{self.event_parent_key}={self.id} and stdout != '' order by start_line) to stdout"  # nosql
-                cursor.copy_expert(sql, fd)
+                # psycopg3's copy writes bytes, but callers of this
+                # function assume a str-based fd will be returned; decode
+                # .write() calls on the fly to maintain this interface
+                with cursor.copy(sql) as copy:
+                    while data := copy.read():
+                        fd.write(smart_str(bytes(data)))
 
                 if hasattr(fd, 'name'):
                     # If we're dealing with a physical file, use `sed` to clean

awx/main/models/workflow.py

@@ -82,7 +82,7 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig):
         related_name='%(class)ss_always',
     )
     all_parents_must_converge = models.BooleanField(
-        default=False, help_text=_("If enabled then the node will only run if all of the parent nodes " "have met the criteria to reach this node")
+        default=False, help_text=_("If enabled then the node will only run if all of the parent nodes have met the criteria to reach this node")
     )
     unified_job_template = models.ForeignKey(
         'UnifiedJobTemplate',
@@ -181,7 +181,7 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
         max_length=512,
         default=uuid4,
         blank=False,
-        help_text=_('An identifier for this node that is unique within its workflow. ' 'It is copied to workflow job nodes corresponding to this node.'),
+        help_text=_('An identifier for this node that is unique within its workflow. It is copied to workflow job nodes corresponding to this node.'),
     )
     instance_groups = OrderedManyToManyField(
         'InstanceGroup',
@@ -334,7 +334,7 @@ class WorkflowJobNode(WorkflowNodeBase):
             accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs(**node_prompts_data)
             if errors:
                 logger.info(
-                    _('Bad launch configuration starting template {template_pk} as part of ' 'workflow {workflow_pk}. Errors:\n{error_text}').format(
+                    _('Bad launch configuration starting template {template_pk} as part of workflow {workflow_pk}. Errors:\n{error_text}').format(
                         template_pk=ujt_obj.pk, workflow_pk=self.pk, error_text=errors
                     )
                 )
@@ -647,7 +647,7 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
         null=True,
         default=None,
         on_delete=models.SET_NULL,
-        help_text=_("If automatically created for a sliced job run, the job template " "the workflow job was created from."),
+        help_text=_("If automatically created for a sliced job run, the job template the workflow job was created from."),
     )
     is_sliced_job = models.BooleanField(default=False)
     is_bulk_job = models.BooleanField(default=False)
@@ -714,7 +714,7 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
         wj = self.get_workflow_job()
         while wj and wj.workflow_job_template_id:
             if wj.pk in wj_ids:
-                logger.critical('Cycles detected in the workflow jobs graph, ' 'this is not normal and suggests task manager degeneracy.')
+                logger.critical('Cycles detected in the workflow jobs graph, this is not normal and suggests task manager degeneracy.')
                 break
             wj_ids.add(wj.pk)
             ancestors.append(wj.workflow_job_template)

awx/main/notifications/custom_notification_base.py

@@ -8,7 +8,7 @@ class CustomNotificationBase(object):
 
     DEFAULT_APPROVAL_RUNNING_MSG = 'The approval node "{{ approval_node_name }}" needs review. This node can be viewed at: {{ workflow_url }}'
     DEFAULT_APPROVAL_RUNNING_BODY = (
-        'The approval node "{{ approval_node_name }}" needs review. ' 'This approval node can be viewed at: {{ workflow_url }}\n\n{{ job_metadata }}'
+        'The approval node "{{ approval_node_name }}" needs review. This approval node can be viewed at: {{ workflow_url }}\n\n{{ job_metadata }}'
     )
 
     DEFAULT_APPROVAL_APPROVED_MSG = 'The approval node "{{ approval_node_name }}" was approved. {{ workflow_url }}'

awx/main/notifications/webhook_backend.py

@@ -32,7 +32,7 @@ class WebhookBackend(AWXBaseEmailBackend, CustomNotificationBase):
         "success": {"body": DEFAULT_BODY},
         "error": {"body": DEFAULT_BODY},
         "workflow_approval": {
-            "running": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" needs review. ' 'This node can be viewed at: {{ workflow_url }}"}'},
+            "running": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" needs review. This node can be viewed at: {{ workflow_url }}"}'},
             "approved": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was approved. {{ workflow_url }}"}'},
             "timed_out": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" has timed out. {{ workflow_url }}"}'},
             "denied": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was denied. {{ workflow_url }}"}'},

awx/main/tasks/jobs.py

@@ -290,13 +290,6 @@ class BaseTask(object):
             content = safe_dump(vars, safe_dict)
         return self.write_private_data_file(private_data_dir, 'extravars', content, sub_dir='env')
 
-    def add_awx_venv(self, env):
-        env['VIRTUAL_ENV'] = settings.AWX_VENV_PATH
-        if 'PATH' in env:
-            env['PATH'] = os.path.join(settings.AWX_VENV_PATH, "bin") + ":" + env['PATH']
-        else:
-            env['PATH'] = os.path.join(settings.AWX_VENV_PATH, "bin")
-
     def build_env(self, instance, private_data_dir, private_data_files=None):
         """
         Build environment dictionary for ansible-playbook.
@@ -926,6 +919,7 @@ class RunJob(SourceControlMixin, BaseTask):
         path_vars = (
             ('ANSIBLE_COLLECTIONS_PATHS', 'collections_paths', 'requirements_collections', '~/.ansible/collections:/usr/share/ansible/collections'),
             ('ANSIBLE_ROLES_PATH', 'roles_path', 'requirements_roles', '~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles'),
+            ('ANSIBLE_COLLECTIONS_PATH', 'collections_path', 'requirements_collections', '~/.ansible/collections:/usr/share/ansible/collections'),
         )
 
         config_values = read_ansible_config(os.path.join(private_data_dir, 'project'), list(map(lambda x: x[1], path_vars)))
@@ -1268,7 +1262,7 @@ class RunProjectUpdate(BaseTask):
 
         galaxy_creds_are_defined = project_update.project.organization and project_update.project.organization.galaxy_credentials.exists()
         if not galaxy_creds_are_defined and (settings.AWX_ROLES_ENABLED or settings.AWX_COLLECTIONS_ENABLED):
-            logger.warning('Galaxy role/collection syncing is enabled, but no ' f'credentials are configured for {project_update.project.organization}.')
+            logger.warning('Galaxy role/collection syncing is enabled, but no credentials are configured for {project_update.project.organization}.')
 
         extra_vars.update(
             {

awx/main/tasks/receptor.py

@@ -643,7 +643,7 @@ RECEPTOR_CONFIG_STARTER = (
     {'node': {'firewallrules': [{'action': 'reject', 'tonode': settings.CLUSTER_HOST_ID, 'toservice': 'control'}]}},
     {'control-service': {'service': 'control', 'filename': '/var/run/receptor/receptor.sock', 'permissions': '0660'}},
     {'work-command': {'worktype': 'local', 'command': 'ansible-runner', 'params': 'worker', 'allowruntimeparams': True}},
-    {'work-signing': {'privatekey': '/etc/receptor/signing/work-private-key.pem', 'tokenexpiration': '1m'}},
+    {'work-signing': {'privatekey': '/etc/receptor/work_private_key.pem', 'tokenexpiration': '1m'}},
     {
         'work-kubernetes': {
             'worktype': 'kubernetes-runtime-auth',
@@ -665,7 +665,7 @@ RECEPTOR_CONFIG_STARTER = (
     {
         'tls-client': {
             'name': 'tlsclient',
-            'rootcas': '/etc/receptor/tls/ca/receptor-ca.crt',
+            'rootcas': '/etc/receptor/tls/ca/mesh-CA.crt',
             'cert': '/etc/receptor/tls/receptor.crt',
             'key': '/etc/receptor/tls/receptor.key',
             'mintls13': False,

awx/main/tasks/system.py

@@ -541,7 +541,7 @@ def cluster_node_heartbeat(dispatch_time=None, worker_tasks=None):
             logger.warning(f'Heartbeat skew - interval={(nowtime - last_last_seen).total_seconds():.4f}, expected={settings.CLUSTER_NODE_HEARTBEAT_PERIOD}')
     else:
         if settings.AWX_AUTO_DEPROVISION_INSTANCES:
-            (changed, this_inst) = Instance.objects.register(ip_address=os.environ.get('MY_POD_IP'), node_type='control', uuid=settings.SYSTEM_UUID)
+            (changed, this_inst) = Instance.objects.register(ip_address=os.environ.get('MY_POD_IP'), node_type='control', node_uuid=settings.SYSTEM_UUID)
             if changed:
                 logger.warning(f'Recreated instance record {this_inst.hostname} after unexpected removal')
             this_inst.local_health_check()

awx/main/tests/docs/test_swagger_generation.py

@@ -7,7 +7,7 @@ from django.core.serializers.json import DjangoJSONEncoder
 from django.utils.functional import Promise
 from django.utils.encoding import force_str
 
-from openapi_codec.encode import generate_swagger_object
+from drf_yasg.codecs import OpenAPICodecJson
 import pytest
 
 from awx.api.versioning import drf_reverse
@@ -43,12 +43,12 @@ class TestSwaggerGeneration:
     @pytest.fixture(autouse=True, scope='function')
     def _prepare(self, get, admin):
         if not self.__class__.JSON:
-            url = drf_reverse('api:swagger_view') + '?format=openapi'
+            url = drf_reverse('api:schema-swagger-ui') + '?format=openapi'
             response = get(url, user=admin)
-            data = generate_swagger_object(response.data)
+            codec = OpenAPICodecJson([])
+            data = codec.generate_swagger_object(response.data)
             if response.has_header('X-Deprecated-Paths'):
                 data['deprecated_paths'] = json.loads(response['X-Deprecated-Paths'])
-            data.update(response.accepted_renderer.get_customizations() or {})
 
             data['host'] = None
             data['schemes'] = ['https']
@@ -60,12 +60,21 @@ class TestSwaggerGeneration:
                 # change {version} in paths to the actual default API version (e.g., v2)
                 revised_paths[path.replace('{version}', settings.REST_FRAMEWORK['DEFAULT_VERSION'])] = node
                 for method in node:
+                    # Ignore any parameters methods, these cause issues because it can come as an array instead of a dict
+                    # Which causes issues in the last for loop in here
+                    if method == 'parameters':
+                        continue
+
                     if path in deprecated_paths:
                         node[method]['deprecated'] = True
                     if 'description' in node[method]:
                         # Pop off the first line and use that as the summary
                         lines = node[method]['description'].splitlines()
-                        node[method]['summary'] = lines.pop(0).strip('#:')
+                        # If there was a description then set the summary as the description, otherwise make something up
+                        if lines:
+                            node[method]['summary'] = lines.pop(0).strip('#:')
+                        else:
+                            node[method]['summary'] = f'No Description for {method} on {path}'
                         node[method]['description'] = '\n'.join(lines)
 
                     # remove the required `version` parameter
@@ -90,13 +99,13 @@ class TestSwaggerGeneration:
         # The number of API endpoints changes over time, but let's just check
         # for a reasonable number here; if this test starts failing, raise/lower the bounds
         paths = JSON['paths']
-        assert 250 < len(paths) < 350
-        assert list(paths['/api/'].keys()) == ['get']
-        assert list(paths['/api/v2/'].keys()) == ['get']
-        assert list(sorted(paths['/api/v2/credentials/'].keys())) == ['get', 'post']
-        assert list(sorted(paths['/api/v2/credentials/{id}/'].keys())) == ['delete', 'get', 'patch', 'put']
-        assert list(paths['/api/v2/settings/'].keys()) == ['get']
-        assert list(paths['/api/v2/settings/{category_slug}/'].keys()) == ['get', 'put', 'patch', 'delete']
+        assert 250 < len(paths) < 375
+        assert set(list(paths['/api/'].keys())) == set(['get', 'parameters'])
+        assert set(list(paths['/api/v2/'].keys())) == set(['get', 'parameters'])
+        assert set(list(sorted(paths['/api/v2/credentials/'].keys()))) == set(['get', 'post', 'parameters'])
+        assert set(list(sorted(paths['/api/v2/credentials/{id}/'].keys()))) == set(['delete', 'get', 'patch', 'put', 'parameters'])
+        assert set(list(paths['/api/v2/settings/'].keys())) == set(['get', 'parameters'])
+        assert set(list(paths['/api/v2/settings/{category_slug}/'].keys())) == set(['get', 'put', 'patch', 'delete', 'parameters'])
 
     @pytest.mark.parametrize(
         'path',
@@ -162,4 +171,8 @@ class TestSwaggerGeneration:
             data = re.sub(r'[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\s)[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+(Z|\+[0-9]{2}:[0-9]{2})?', r'2018-02-01T08:00:00.000000Z', data)
             data = re.sub(r'''(\s+"client_id": ")([a-zA-Z0-9]{40})("\,\s*)''', r'\1xxxx\3', data)
             data = re.sub(r'"action_node": "[^"]+"', '"action_node": "awx"', data)
+
+            # replace uuids to prevent needless diffs
+            pattern = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+            data = re.sub(pattern, r'00000000-0000-0000-0000-000000000000', data)
             f.write(data)

awx/main/tests/functional/analytics/test_collectors.py

@@ -2,8 +2,8 @@ import pytest
 import tempfile
 import os
 import re
-import shutil
 import csv
+from io import StringIO
 
 from django.utils.timezone import now
 from datetime import timedelta
@@ -20,15 +20,16 @@ from awx.main.models import (
 )
 
 
-@pytest.fixture
-def sqlite_copy_expert(request):
-    # copy_expert is postgres-specific, and SQLite doesn't support it; mock its
-    # behavior to test that it writes a file that contains stdout from events
-    path = tempfile.mkdtemp(prefix="copied_tables")
+class MockCopy:
+    headers = None
+    results = None
+    sent_data = False
 
-    def write_stdout(self, sql, fd):
+    def __init__(self, sql, parent_connection):
         # Would be cool if we instead properly disected the SQL query and verified
         # it that way. But instead, we just take the naive approach here.
+        self.results = None
+        self.headers = None
         sql = sql.strip()
         assert sql.startswith("COPY (")
         assert sql.endswith(") TO STDOUT WITH CSV HEADER")
@@ -51,29 +52,49 @@ def sqlite_copy_expert(request):
             elif not line.endswith(","):
                 sql_new[-1] = sql_new[-1].rstrip(",")
         sql = "\n".join(sql_new)
+        parent_connection.execute(sql)
+        self.results = parent_connection.fetchall()
+        self.headers = [i[0] for i in parent_connection.description]
 
-        self.execute(sql)
-        results = self.fetchall()
-        headers = [i[0] for i in self.description]
+    def read(self):
+        if not self.sent_data:
+            mem_file = StringIO()
+            csv_handle = csv.writer(
+                mem_file,
+                delimiter=",",
+                quoting=csv.QUOTE_ALL,
+                escapechar="\\",
+                lineterminator="\n",
+            )
+            if self.headers:
+                csv_handle.writerow(self.headers)
+            if self.results:
+                csv_handle.writerows(self.results)
+            self.sent_data = True
+            return memoryview((mem_file.getvalue()).encode())
+        return None
 
-        csv_handle = csv.writer(
-            fd,
-            delimiter=",",
-            quoting=csv.QUOTE_ALL,
-            escapechar="\\",
-            lineterminator="\n",
-        )
-        csv_handle.writerow(headers)
-        csv_handle.writerows(results)
+    def __enter__(self):
+        return self
 
-    setattr(SQLiteCursorWrapper, "copy_expert", write_stdout)
-    request.addfinalizer(lambda: shutil.rmtree(path))
-    request.addfinalizer(lambda: delattr(SQLiteCursorWrapper, "copy_expert"))
-    return path
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+@pytest.fixture
+def sqlite_copy(request, mocker):
+    # copy is postgres-specific, and SQLite doesn't support it; mock its
+    # behavior to test that it writes a file that contains stdout from events
+
+    def write_stdout(self, sql):
+        mock_copy = MockCopy(sql, self)
+        return mock_copy
+
+    mocker.patch.object(SQLiteCursorWrapper, 'copy', write_stdout, create=True)
 
 
 @pytest.mark.django_db
-def test_copy_tables_unified_job_query(sqlite_copy_expert, project, inventory, job_template):
+def test_copy_tables_unified_job_query(sqlite_copy, project, inventory, job_template):
     """
     Ensure that various unified job types are in the output of the query.
     """
@@ -127,7 +148,7 @@ def workflow_job(states=["new", "new", "new", "new", "new"]):
 
 
 @pytest.mark.django_db
-def test_copy_tables_workflow_job_node_query(sqlite_copy_expert, workflow_job):
+def test_copy_tables_workflow_job_node_query(sqlite_copy, workflow_job):
     time_start = now() - timedelta(hours=9)
 
     with tempfile.TemporaryDirectory() as tmpdir:

awx/main/tests/functional/api/test_credential.py

@@ -77,7 +77,7 @@ def test_credential_validation_error_with_multiple_owner_fields(post, admin, ali
     }
     response = post(reverse('api:credential_list'), params, admin)
     assert response.status_code == 400
-    assert response.data['detail'][0] == ("Only one of 'user', 'team', or 'organization' should be provided, " "received organization, team, user fields.")
+    assert response.data['detail'][0] == ("Only one of 'user', 'team', or 'organization' should be provided, received organization, team, user fields.")
 
 
 @pytest.mark.django_db
@@ -925,7 +925,7 @@ def test_credential_type_mutability(patch, organization, admin, credentialtype_s
 
     response = _change_credential_type()
     assert response.status_code == 400
-    expected = ['You cannot change the credential type of the credential, ' 'as it may break the functionality of the resources using it.']
+    expected = ['You cannot change the credential type of the credential, as it may break the functionality of the resources using it.']
     assert response.data['credential_type'] == expected
 
     response = patch(reverse('api:credential_detail', kwargs={'pk': cred.pk}), {'name': 'Worst credential ever'}, admin)
@@ -962,7 +962,7 @@ def test_vault_credential_type_mutability(patch, organization, admin, credential
 
     response = _change_credential_type()
     assert response.status_code == 400
-    expected = ['You cannot change the credential type of the credential, ' 'as it may break the functionality of the resources using it.']
+    expected = ['You cannot change the credential type of the credential, as it may break the functionality of the resources using it.']
     assert response.data['credential_type'] == expected
 
     response = patch(reverse('api:credential_detail', kwargs={'pk': cred.pk}), {'name': 'Worst credential ever'}, admin)
@@ -994,7 +994,7 @@ def test_cloud_credential_type_mutability(patch, organization, admin, credential
 
     response = _change_credential_type()
     assert response.status_code == 400
-    expected = ['You cannot change the credential type of the credential, ' 'as it may break the functionality of the resources using it.']
+    expected = ['You cannot change the credential type of the credential, as it may break the functionality of the resources using it.']
     assert response.data['credential_type'] == expected
 
     response = patch(reverse('api:credential_detail', kwargs={'pk': cred.pk}), {'name': 'Worst credential ever'}, admin)

awx/main/tests/functional/api/test_job.py

@@ -51,6 +51,16 @@ def test_job_relaunch_permission_denied_response(post, get, inventory, project,
     r = post(reverse('api:job_relaunch', kwargs={'pk': job.pk}), {}, jt_user, expect=201)
 
 
+@pytest.mark.django_db
+def test_label_sublist(get, admin_user, organization):
+    job = Job.objects.create()
+    label = Label.objects.create(organization=organization, name='Steve')
+    job.labels.add(label)
+    r = get(url=reverse('api:job_label_list', kwargs={'pk': job.pk}), user=admin_user, expect=200)
+    assert r.data['count'] == 1
+    assert r.data['results'].pop()['id'] == label.id
+
+
 @pytest.mark.django_db
 def test_job_relaunch_prompts_not_accepted_response(post, get, inventory, project, credential, net_credential, machine_credential):
     jt = JobTemplate.objects.create(name='testjt', inventory=inventory, project=project)
@@ -214,7 +224,7 @@ class TestControllerNode:
         return AdHocCommand.objects.create(inventory=inventory)
 
     @pytest.mark.django_db
-    def test_field_controller_node_exists(self, sqlite_copy_expert, admin_user, job, project_update, inventory_update, adhoc, get, system_job_factory):
+    def test_field_controller_node_exists(self, sqlite_copy, admin_user, job, project_update, inventory_update, adhoc, get, system_job_factory):
         system_job = system_job_factory()
 
         r = get(reverse('api:unified_job_list') + '?id={}'.format(job.id), admin_user, expect=200)

awx/main/tests/functional/api/test_unified_jobs_stdout.py

@@ -57,7 +57,7 @@ def _mk_inventory_update(created=None):
         [_mk_inventory_update, InventoryUpdateEvent, 'inventory_update', 'api:inventory_update_stdout'],
     ],
 )
-def test_text_stdout(sqlite_copy_expert, Parent, Child, relation, view, get, admin):
+def test_text_stdout(sqlite_copy, Parent, Child, relation, view, get, admin):
     job = Parent()
     job.save()
     for i in range(3):
@@ -79,7 +79,7 @@ def test_text_stdout(sqlite_copy_expert, Parent, Child, relation, view, get, adm
     ],
 )
 @pytest.mark.parametrize('download', [True, False])
-def test_ansi_stdout_filtering(sqlite_copy_expert, Parent, Child, relation, view, download, get, admin):
+def test_ansi_stdout_filtering(sqlite_copy, Parent, Child, relation, view, download, get, admin):
     job = Parent()
     job.save()
     for i in range(3):
@@ -111,7 +111,7 @@ def test_ansi_stdout_filtering(sqlite_copy_expert, Parent, Child, relation, view
         [_mk_inventory_update, InventoryUpdateEvent, 'inventory_update', 'api:inventory_update_stdout'],
     ],
 )
-def test_colorized_html_stdout(sqlite_copy_expert, Parent, Child, relation, view, get, admin):
+def test_colorized_html_stdout(sqlite_copy, Parent, Child, relation, view, get, admin):
     job = Parent()
     job.save()
     for i in range(3):
@@ -134,7 +134,7 @@ def test_colorized_html_stdout(sqlite_copy_expert, Parent, Child, relation, view
         [_mk_inventory_update, InventoryUpdateEvent, 'inventory_update', 'api:inventory_update_stdout'],
     ],
 )
-def test_stdout_line_range(sqlite_copy_expert, Parent, Child, relation, view, get, admin):
+def test_stdout_line_range(sqlite_copy, Parent, Child, relation, view, get, admin):
     job = Parent()
     job.save()
     for i in range(20):
@@ -146,7 +146,7 @@ def test_stdout_line_range(sqlite_copy_expert, Parent, Child, relation, view, ge
 
 
 @pytest.mark.django_db
-def test_text_stdout_from_system_job_events(sqlite_copy_expert, get, admin):
+def test_text_stdout_from_system_job_events(sqlite_copy, get, admin):
     created = tz_now()
     job = SystemJob(created=created)
     job.save()
@@ -158,7 +158,7 @@ def test_text_stdout_from_system_job_events(sqlite_copy_expert, get, admin):
 
 
 @pytest.mark.django_db
-def test_text_stdout_with_max_stdout(sqlite_copy_expert, get, admin):
+def test_text_stdout_with_max_stdout(sqlite_copy, get, admin):
     created = tz_now()
     job = SystemJob(created=created)
     job.save()
@@ -185,7 +185,7 @@ def test_text_stdout_with_max_stdout(sqlite_copy_expert, get, admin):
 )
 @pytest.mark.parametrize('fmt', ['txt', 'ansi'])
 @mock.patch('awx.main.redact.UriCleaner.SENSITIVE_URI_PATTERN', mock.Mock(**{'search.return_value': None}))  # really slow for large strings
-def test_max_bytes_display(sqlite_copy_expert, Parent, Child, relation, view, fmt, get, admin):
+def test_max_bytes_display(sqlite_copy, Parent, Child, relation, view, fmt, get, admin):
     created = tz_now()
     job = Parent(created=created)
     job.save()
@@ -255,7 +255,7 @@ def test_legacy_result_stdout_with_max_bytes(Cls, view, fmt, get, admin):
     ],
 )
 @pytest.mark.parametrize('fmt', ['txt', 'ansi', 'txt_download', 'ansi_download'])
-def test_text_with_unicode_stdout(sqlite_copy_expert, Parent, Child, relation, view, get, admin, fmt):
+def test_text_with_unicode_stdout(sqlite_copy, Parent, Child, relation, view, get, admin, fmt):
     job = Parent()
     job.save()
     for i in range(3):
@@ -267,7 +267,7 @@ def test_text_with_unicode_stdout(sqlite_copy_expert, Parent, Child, relation, v
 
 
 @pytest.mark.django_db
-def test_unicode_with_base64_ansi(sqlite_copy_expert, get, admin):
+def test_unicode_with_base64_ansi(sqlite_copy, get, admin):
     created = tz_now()
     job = Job(created=created)
     job.save()

awx/main/tests/functional/conftest.py

@@ -1,8 +1,6 @@
 # Python
 import pytest
 from unittest import mock
-import tempfile
-import shutil
 import urllib.parse
 from unittest.mock import PropertyMock
 
@@ -789,25 +787,43 @@ def oauth_application(admin):
     return Application.objects.create(name='test app', user=admin, client_type='confidential', authorization_grant_type='password')
 
 
-@pytest.fixture
-def sqlite_copy_expert(request):
-    # copy_expert is postgres-specific, and SQLite doesn't support it; mock its
-    # behavior to test that it writes a file that contains stdout from events
-    path = tempfile.mkdtemp(prefix='job-event-stdout')
+class MockCopy:
+    events = []
+    index = -1
 
-    def write_stdout(self, sql, fd):
-        # simulate postgres copy_expert support with ORM code
+    def __init__(self, sql):
+        self.events = []
         parts = sql.split(' ')
         tablename = parts[parts.index('from') + 1]
         for cls in (JobEvent, AdHocCommandEvent, ProjectUpdateEvent, InventoryUpdateEvent, SystemJobEvent):
             if cls._meta.db_table == tablename:
                 for event in cls.objects.order_by('start_line').all():
-                    fd.write(event.stdout)
+                    self.events.append(event.stdout)
 
-    setattr(SQLiteCursorWrapper, 'copy_expert', write_stdout)
-    request.addfinalizer(lambda: shutil.rmtree(path))
-    request.addfinalizer(lambda: delattr(SQLiteCursorWrapper, 'copy_expert'))
-    return path
+    def read(self):
+        self.index = self.index + 1
+        if self.index < len(self.events):
+            return memoryview(self.events[self.index].encode())
+
+        return None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+@pytest.fixture
+def sqlite_copy(request, mocker):
+    # copy is postgres-specific, and SQLite doesn't support it; mock its
+    # behavior to test that it writes a file that contains stdout from events
+
+    def write_stdout(self, sql):
+        mock_copy = MockCopy(sql)
+        return mock_copy
+
+    mocker.patch.object(SQLiteCursorWrapper, 'copy', write_stdout, create=True)
 
 
 @pytest.fixture

awx/main/tests/functional/models/test_inventory.py

@@ -271,6 +271,7 @@ def test_inventory_update_excessively_long_name(inventory, inventory_source):
 class TestHostManager:
     def test_host_filter_not_smart(self, setup_ec2_gce, organization):
         smart_inventory = Inventory(name='smart', organization=organization, host_filter='inventory_sources__source=ec2')
+        smart_inventory.save()
         assert len(smart_inventory.hosts.all()) == 0
 
     def test_host_distinctness(self, setup_inventory_groups, organization):

awx/main/tests/functional/models/test_notifications.py

@@ -98,7 +98,7 @@ class TestJobNotificationMixin(object):
 
     @pytest.mark.django_db
     @pytest.mark.parametrize('JobClass', [AdHocCommand, InventoryUpdate, Job, ProjectUpdate, SystemJob, WorkflowJob])
-    def test_context(self, JobClass, sqlite_copy_expert, project, inventory_source):
+    def test_context(self, JobClass, sqlite_copy, project, inventory_source):
         """The Jinja context defines all of the fields that can be used by a template. Ensure that the context generated
         for each job type has the expected structure."""
         kwargs = {}

awx/main/tests/functional/test_credential.py

@@ -78,6 +78,7 @@ def test_default_cred_types():
         [
             'aim',
             'aws',
+            'aws_secretsmanager_credential',
             'azure_kv',
             'azure_rm',
             'centrify_vault_kv',

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -121,7 +121,7 @@ def read_content(private_data_dir, raw_env, inventory_update):
                     break
                 alias = 'file_reference_{}'.format(i)
             else:
-                raise RuntimeError('Test not able to cope with >10 references by env vars. ' 'Something probably went very wrong.')
+                raise RuntimeError('Test not able to cope with >10 references by env vars. Something probably went very wrong.')
             file_aliases[abs_file_path] = alias
             for env_key in inverse_env[runner_path]:
                 env[env_key] = '{{{{ {} }}}}'.format(alias)
@@ -234,7 +234,7 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential
             source_dir = os.path.join(base_dir, this_kind)  # this_kind is a global
 
             if not os.path.exists(source_dir):
-                raise FileNotFoundError('Maybe you never made reference files? ' 'MAKE_INVENTORY_REFERENCE_FILES=true py.test ...\noriginal: {}')
+                raise FileNotFoundError('Maybe you never made reference files? MAKE_INVENTORY_REFERENCE_FILES=true py.test ...\noriginal: {}')
             files_dir = os.path.join(source_dir, 'files')
             try:
                 expected_file_list = os.listdir(files_dir)

awx/main/tests/functional/test_notifications.py

@@ -16,8 +16,7 @@ from django.test.utils import override_settings
 @pytest.mark.django_db
 def test_get_notification_template_list(get, user, notification_template):
     url = reverse('api:notification_template_list')
-    response = get(url, user('admin', True))
-    assert response.status_code == 200
+    response = get(url, user('admin', True), expect=200)
     assert len(response.data['results']) == 1
 
 
@@ -35,8 +34,8 @@ def test_basic_parameterization(get, post, user, organization):
             notification_configuration=dict(url="http://localhost", disable_ssl_verification=False, headers={"Test": "Header"}),
         ),
         u,
+        expect=201,
     )
-    assert response.status_code == 201
     url = reverse('api:notification_template_detail', kwargs={'pk': response.data['id']})
     response = get(url, u)
     assert 'related' in response.data
@@ -69,8 +68,8 @@ def test_encrypted_subfields(get, post, user, organization):
             notification_configuration=dict(account_sid="dummy", account_token="shouldhide", from_number="+19999999999", to_numbers=["9998887777"]),
         ),
         u,
+        expect=201,
     )
-    assert response.status_code == 201
     notification_template_actual = NotificationTemplate.objects.get(id=response.data['id'])
     url = reverse('api:notification_template_detail', kwargs={'pk': response.data['id']})
     response = get(url, u)
@@ -96,8 +95,8 @@ def test_inherited_notification_templates(get, post, user, organization, project
                 notification_configuration=dict(url="http://localhost", disable_ssl_verification=False, headers={"Test": "Header"}),
             ),
             u,
+            expect=201,
         )
-        assert response.status_code == 201
         notification_templates.append(response.data['id'])
     i = Inventory.objects.create(name='test', organization=organization)
     i.save()
@@ -122,8 +121,7 @@ def test_disallow_delete_when_notifications_pending(delete, user, notification_t
     u = user('superuser', True)
     url = reverse('api:notification_template_detail', kwargs={'pk': notification_template.id})
     Notification.objects.create(notification_template=notification_template, status='pending')
-    response = delete(url, user=u)
-    assert response.status_code == 405
+    delete(url, user=u, expect=405)
 
 
 @pytest.mark.django_db
@@ -133,9 +131,8 @@ def test_notification_template_list_includes_notification_errors(get, user, noti
     Notification.objects.create(notification_template=notification_template, status='successful')
     url = reverse('api:notification_template_list')
     u = user('superuser', True)
-    response = get(url, user=u)
+    response = get(url, user=u, expect=200)
 
-    assert response.status_code == 200
     notifications = response.data['results'][0]['summary_fields']['recent_notifications']
     assert len(notifications) == 3
     statuses = [n['status'] for n in notifications]
@@ -163,8 +160,8 @@ def test_custom_environment_injection(post, user, organization):
             notification_configuration=dict(url="https://example.org", disable_ssl_verification=False, http_method="POST", headers={"Test": "Header"}),
         ),
         u,
+        expect=201,
     )
-    assert response.status_code == 201
     template = NotificationTemplate.objects.get(pk=response.data['id'])
     with pytest.raises(ConnectionError), override_settings(AWX_TASK_ENV={'HTTPS_PROXY': '192.168.50.100:1234'}), mock.patch.object(
         HTTPAdapter, 'send'

awx/main/tests/functional/test_projects.py

@@ -4,7 +4,7 @@ from unittest import mock  # noqa
 import pytest
 
 from awx.api.versioning import reverse
-from awx.main.models import Project
+from awx.main.models import Project, JobTemplate
 
 from django.core.exceptions import ValidationError
 
@@ -451,3 +451,19 @@ def test_project_list_ordering_with_duplicate_names(get, order_by, organization_
         results = get(reverse('api:project_list'), objects.superusers.admin, QUERY_STRING='order_by=%s' % order_by).data['results']
         project_ids[x] = [proj['id'] for proj in results]
     assert project_ids[0] == project_ids[1] == project_ids[2] == [1, 2, 3, 4, 5]
+
+
+@pytest.mark.django_db
+def test_project_failed_update(post, project, admin, inventory):
+    """Test to ensure failed projects with update on launch will create launch rather than error"""
+    jt = JobTemplate.objects.create(project=project, inventory=inventory)
+    # set project to update on launch and set status to failed
+    project.update_fields(scm_update_on_launch=True)
+    project.update()
+    project.project_updates.last().update_fields(status='failed')
+    response = post(reverse('api:job_template_launch', kwargs={'pk': jt.pk}), user=admin, expect=201)
+    assert response.status_code == 201
+    # set project to not update on launch and validate still 400's
+    project.update_fields(scm_update_on_launch=False)
+    response = post(reverse('api:job_template_launch', kwargs={'pk': jt.pk}), user=admin, expect=400)
+    assert response.status_code == 400

awx/main/tests/unit/api/test_logger.py

@@ -47,7 +47,7 @@ data_loggly = {
             '\n'.join(
                 [
                     'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                    'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")',  # noqa
+                    'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")',  # noqa
                 ]
             ),
         ),
@@ -89,7 +89,7 @@ data_loggly = {
             '\n'.join(
                 [
                     'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                    'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
+                    'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
                 ]
             ),
         ),
@@ -103,7 +103,7 @@ data_loggly = {
             '\n'.join(
                 [
                     'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                    'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
+                    'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
                 ]
             ),
         ),
@@ -117,7 +117,7 @@ data_loggly = {
             '\n'.join(
                 [
                     'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                    'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
+                    'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
                 ]
             ),
         ),
@@ -131,7 +131,7 @@ data_loggly = {
             '\n'.join(
                 [
                     'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                    'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
+                    'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
                 ]
             ),
         ),
@@ -145,7 +145,7 @@ data_loggly = {
             '\n'.join(
                 [
                     'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                    'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
+                    'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
                 ]
             ),
         ),
@@ -159,7 +159,7 @@ data_loggly = {
             '\n'.join(
                 [
                     'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                    'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
+                    'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
                 ]
             ),
         ),
@@ -173,7 +173,7 @@ data_loggly = {
             '\n'.join(
                 [
                     'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                    'action(type="omhttp" server="endpoint5.collection.us2.sumologic.com" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="receiver/v1/http/ZaVnC4dhaV0qoiETY0MrM3wwLoDgO1jFgjOxE6-39qokkj3LGtOroZ8wNaN2M6DtgYrJZsmSi4-36_Up5TbbN_8hosYonLKHSSOSKY845LuLZBCBwStrHQ==")',  # noqa
+                    'action(type="omhttp" server="endpoint5.collection.us2.sumologic.com" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="receiver/v1/http/ZaVnC4dhaV0qoiETY0MrM3wwLoDgO1jFgjOxE6-39qokkj3LGtOroZ8wNaN2M6DtgYrJZsmSi4-36_Up5TbbN_8hosYonLKHSSOSKY845LuLZBCBwStrHQ==")',  # noqa
                 ]
             ),
         ),

awx/main/tests/unit/models/test_events.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from django.utils.timezone import utc
+from datetime import timezone
 import pytest
 
 from awx.main.models import JobEvent, ProjectUpdateEvent, AdHocCommandEvent, InventoryUpdateEvent, SystemJobEvent
@@ -18,7 +18,7 @@ from awx.main.models import JobEvent, ProjectUpdateEvent, AdHocCommandEvent, Inv
 @pytest.mark.parametrize('created', [datetime(2018, 1, 1).isoformat(), datetime(2018, 1, 1)])
 def test_event_parse_created(job_identifier, cls, created):
     event = cls.create_from_data(**{job_identifier: 123, 'created': created})
-    assert event.created == datetime(2018, 1, 1).replace(tzinfo=utc)
+    assert event.created == datetime(2018, 1, 1).replace(tzinfo=timezone.utc)
 
 
 @pytest.mark.parametrize(

awx/main/tests/unit/test_tasks.py

@@ -371,7 +371,7 @@ class TestExtraVarSanitation(TestJobExecution):
     # are deemed trustable, because they can only be added by users w/ enough
     # privilege to add/modify a Job Template)
 
-    UNSAFE = '{{ lookup(' 'pipe' ',' 'ls -la' ') }}'
+    UNSAFE = "{{ lookup('pipe', 'ls -la') }}"
 
     def test_vars_unsafe_by_default(self, job, private_data_dir, mock_me):
         job.created_by = User(pk=123, username='angry-spud')

awx/main/tests/unit/test_views.py

@@ -88,6 +88,6 @@ def test_global_creation_always_possible(all_views):
                 creatable_view = View
         if not creatable or not global_view:
             continue
-        assert 'POST' in global_view().allowed_methods, 'Resource {} should be creatable in global list view {}. ' 'Can be created now in {}'.format(
+        assert 'POST' in global_view().allowed_methods, 'Resource {} should be creatable in global list view {}. Can be created now in {}'.format(
             model, global_view, creatable_view
         )

awx/main/tests/unit/utils/test_filters.py

@@ -93,7 +93,7 @@ class TestSmartFilterQueryFromString:
     @pytest.mark.parametrize(
         "filter_string",
         [
-            'ansible_facts__facts__facts__blank=' 'ansible_facts__a__b__c__ space  =ggg',
+            'ansible_facts__facts__facts__blank=ansible_facts__a__b__c__ space  =ggg',
         ],
     )
     def test_invalid_filter_strings(self, mock_get_host_model, filter_string):
@@ -104,7 +104,7 @@ class TestSmartFilterQueryFromString:
     @pytest.mark.parametrize(
         "filter_string",
         [
-            'created_by__password__icontains=pbkdf2' 'search=foo or created_by__password__icontains=pbkdf2',
+            'created_by__password__icontains=pbkdf2search=foo or created_by__password__icontains=pbkdf2',
             'created_by__password__icontains=pbkdf2 or search=foo',
         ],
     )

awx/main/utils/common.py

@@ -90,6 +90,7 @@ __all__ = [
     'get_event_partition_epoch',
     'cleanup_new_process',
     'log_excess_runtime',
+    'unified_job_class_to_event_table_name',
 ]
 
 
@@ -716,7 +717,7 @@ def parse_yaml_or_json(vars_str, silent_failure=True):
             if silent_failure:
                 return {}
             raise ParseError(
-                _('Cannot parse as JSON (error: {json_error}) or ' 'YAML (error: {yaml_error}).').format(json_error=str(json_err), yaml_error=str(yaml_err))
+                _('Cannot parse as JSON (error: {json_error}) or YAML (error: {yaml_error}).').format(json_error=str(json_err), yaml_error=str(yaml_err))
             )
     return vars_dict
 
@@ -1219,3 +1220,7 @@ def log_excess_runtime(func_logger, cutoff=5.0, debug_cutoff=5.0, msg=None, add_
         return _new_func
 
     return log_excess_runtime_decorator
+
+
+def unified_job_class_to_event_table_name(job_class):
+    return f'main_{job_class().event_class.__name__.lower()}'

awx/main/utils/external_logging.py

@@ -17,7 +17,8 @@ def construct_rsyslog_conf_template(settings=settings):
     port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
     protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '')
     timeout = getattr(settings, 'LOG_AGGREGATOR_TCP_TIMEOUT', 5)
-    max_disk_space = getattr(settings, 'LOG_AGGREGATOR_MAX_DISK_USAGE_GB', 1)
+    max_disk_space_main_queue = getattr(settings, 'LOG_AGGREGATOR_MAX_DISK_USAGE_GB', 1)
+    max_disk_space_action_queue = getattr(settings, 'LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB', 1)
     spool_directory = getattr(settings, 'LOG_AGGREGATOR_MAX_DISK_USAGE_PATH', '/var/lib/awx').rstrip('/')
     error_log_file = getattr(settings, 'LOG_AGGREGATOR_RSYSLOGD_ERROR_LOG_FILE', '')
 
@@ -32,7 +33,7 @@ def construct_rsyslog_conf_template(settings=settings):
             '$WorkDirectory /var/lib/awx/rsyslog',
             f'$MaxMessageSize {max_bytes}',
             '$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf',
-            f'main_queue(queue.spoolDirectory="{spool_directory}" queue.maxdiskspace="{max_disk_space}g" queue.type="Disk" queue.filename="awx-external-logger-backlog")',  # noqa
+            f'main_queue(queue.spoolDirectory="{spool_directory}" queue.maxdiskspace="{max_disk_space_main_queue}g" queue.type="Disk" queue.filename="awx-external-logger-backlog")',  # noqa
             'module(load="imuxsock" SysSock.Use="off")',
             'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on" RateLimit.Burst="0")',
             'template(name="awx" type="string" string="%rawmsg-after-pri%")',
@@ -78,6 +79,11 @@ def construct_rsyslog_conf_template(settings=settings):
             'action.resumeRetryCount="-1"',
             'template="awx"',
             f'action.resumeInterval="{timeout}"',
+            f'queue.spoolDirectory="{spool_directory}"',
+            'queue.filename="awx-external-logger-action-queue"',
+            f'queue.maxdiskspace="{max_disk_space_action_queue}g"',
+            'queue.type="LinkedList"',
+            'queue.saveOnShutdown="on"',
         ]
         if error_log_file:
             params.append(f'errorfile="{error_log_file}"')

awx/main/utils/mem_inventory.py

@@ -253,7 +253,7 @@ def dict_to_mem_data(data, inventory=None):
                     if isinstance(hv, dict):
                         host.variables.update(hv)
                     else:
-                        logger.warning('Expected dict of vars for ' 'host "%s", got %s instead', hk, str(type(hv)))
+                        logger.warning('Expected dict of vars for host "%s", got %s instead', hk, str(type(hv)))
                     group.add_host(host)
             elif isinstance(hosts, (list, tuple)):
                 for hk in hosts:
@@ -262,13 +262,13 @@ def dict_to_mem_data(data, inventory=None):
                         continue
                     group.add_host(host)
             else:
-                logger.warning('Expected dict or list of "hosts" for ' 'group "%s", got %s instead', k, str(type(hosts)))
+                logger.warning('Expected dict or list of "hosts" for group "%s", got %s instead', k, str(type(hosts)))
             # Process group variables.
             vars = v.get('vars', {})
             if isinstance(vars, dict):
                 group.variables.update(vars)
             else:
-                logger.warning('Expected dict of vars for ' 'group "%s", got %s instead', k, str(type(vars)))
+                logger.warning('Expected dict of vars for group "%s", got %s instead', k, str(type(vars)))
             # Process child groups.
             children = v.get('children', [])
             if isinstance(children, (list, tuple)):
@@ -277,7 +277,7 @@ def dict_to_mem_data(data, inventory=None):
                     if child and c != 'ungrouped':
                         group.add_child_group(child)
             else:
-                logger.warning('Expected list of children for ' 'group "%s", got %s instead', k, str(type(children)))
+                logger.warning('Expected list of children for group "%s", got %s instead', k, str(type(children)))
 
         # Load host names from a list.
         elif isinstance(v, (list, tuple)):
@@ -288,7 +288,7 @@ def dict_to_mem_data(data, inventory=None):
                 group.add_host(host)
         else:
             logger.warning('')
-            logger.warning('Expected dict or list for group "%s", ' 'got %s instead', k, str(type(v)))
+            logger.warning('Expected dict or list for group "%s", got %s instead', k, str(type(v)))
 
         if k not in ['all', 'ungrouped']:
             inventory.all_group.add_child_group(group)
@@ -299,6 +299,6 @@ def dict_to_mem_data(data, inventory=None):
             if isinstance(meta_hostvars, dict):
                 v.variables.update(meta_hostvars)
             else:
-                logger.warning('Expected dict of vars for ' 'host "%s", got %s instead', k, str(type(meta_hostvars)))
+                logger.warning('Expected dict of vars for host "%s", got %s instead', k, str(type(meta_hostvars)))
 
     return inventory

awx/main/wsrelay.py

@@ -5,6 +5,7 @@ from typing import Dict
 
 import aiohttp
 from aiohttp import client_exceptions
+import aioredis
 
 from channels.layers import get_channel_layer
 
@@ -180,6 +181,9 @@ class WebsocketRelayConnection:
                         return
 
                     continue
+                except aioredis.errors.ConnectionClosedError:
+                    logger.info(f"Producer {name} lost connection to Redis, shutting down.")
+                    return
 
                 await websocket.send_json(wrap_broadcast_msg(group, msg))
         except ConnectionResetError:
@@ -205,47 +209,85 @@ class WebSocketRelayManager(object):
         # hostname -> ip
         self.known_hosts: Dict[str, str] = dict()
 
-    async def pg_consumer(self, conn):
-        try:
-            await conn.execute("LISTEN web_heartbeet")
-            async for notif in conn.notifies():
-                if notif is not None and notif.channel == "web_heartbeet":
-                    try:
-                        payload = json.loads(notif.payload)
-                    except json.JSONDecodeError:
-                        logmsg = "Failed to decode message from pg_notify channel `web_heartbeet`"
-                        if logger.isEnabledFor(logging.DEBUG):
-                            logmsg = "{} {}".format(logmsg, payload)
+    async def on_ws_heartbeat(self, conn):
+        await conn.execute("LISTEN web_ws_heartbeat")
+        async for notif in conn.notifies():
+            if notif is None:
+                continue
+            try:
+                if not notif.payload or notif.channel != "web_ws_heartbeat":
+                    return
+
+                try:
+                    payload = json.loads(notif.payload)
+                except json.JSONDecodeError:
+                    logmsg = "Failed to decode message from pg_notify channel `web_ws_heartbeat`"
+                    if logger.isEnabledFor(logging.DEBUG):
+                        logmsg = "{} {}".format(logmsg, payload)
                         logger.warning(logmsg)
-                        continue
+                    return
 
-                    # Skip if the message comes from the same host we are running on
-                    # In this case, we'll be sharing a redis, no need to relay.
-                    if payload.get("hostname") == self.local_hostname:
-                        continue
+                # Skip if the message comes from the same host we are running on
+                # In this case, we'll be sharing a redis, no need to relay.
+                if payload.get("hostname") == self.local_hostname:
+                    return
 
-                    if payload.get("action") == "online":
-                        hostname = payload["hostname"]
-                        ip = payload["ip"]
-                        if ip is None:
-                            # If we don't get an IP, just try the hostname, maybe it resolves
-                            ip = hostname
-                        self.known_hosts[hostname] = ip
-                        logger.debug(f"Web host {hostname} ({ip}) online heartbeat received.")
-                    elif payload.get("action") == "offline":
-                        hostname = payload["hostname"]
-                        del self.known_hosts[hostname]
-                        logger.debug(f"Web host {hostname} ({ip}) offline heartbeat received.")
-        except Exception as e:
-            # This catch-all is the same as the one above. asyncio will eat the exception
-            # but we want to know about it.
-            logger.exception(f"pg_consumer exception: {e}")
+                action = payload.get("action")
+
+                if action in ("online", "offline"):
+                    hostname = payload.get("hostname")
+                    ip = payload.get("ip") or hostname  # try back to hostname if ip isn't supplied
+                    if ip is None:
+                        logger.warning(f"Received invalid {action} ws_heartbeat, missing hostname and ip: {payload}")
+                        return
+                    logger.debug(f"Web host {hostname} ({ip}) {action} heartbeat received.")
+
+                if action == "online":
+                    self.known_hosts[hostname] = ip
+                elif action == "offline":
+                    await self.cleanup_offline_host(hostname)
+            except Exception as e:
+                # This catch-all is the same as the one above. asyncio will eat the exception
+                # but we want to know about it.
+                logger.exception(f"on_ws_heartbeat exception: {e}")
+
+    async def cleanup_offline_host(self, hostname):
+        """
+        Given a hostname, try to cancel its task/connection and remove it from
+        the list of hosts we know about.
+        If the host isn't in the list, assume that it was already deleted and
+        don't error.
+        """
+        if hostname in self.relay_connections:
+            self.relay_connections[hostname].cancel()
+
+            # Wait for the task to actually run its cancel/completion logic
+            # otherwise it might get GC'd too early when we del it below.
+            # Being GC'd too early could generate a scary message in logs:
+            # "Task was destroyed but it is pending!"
+            try:
+                await asyncio.wait_for(self.relay_connections[hostname].async_task, timeout=10)
+            except asyncio.TimeoutError:
+                logger.warning(f"Tried to cancel relay connection for {hostname} but it timed out during cleanup.")
+            except asyncio.CancelledError:
+                # Handle the case where the task was already cancelled by the time we got here.
+                pass
+
+            del self.relay_connections[hostname]
+
+        if hostname in self.known_hosts:
+            del self.known_hosts[hostname]
+
+        try:
+            self.stats_mgr.delete_remote_host_stats(hostname)
+        except KeyError:
+            pass
 
     async def run(self):
         event_loop = asyncio.get_running_loop()
 
-        stats_mgr = RelayWebsocketStatsManager(event_loop, self.local_hostname)
-        stats_mgr.start()
+        self.stats_mgr = RelayWebsocketStatsManager(event_loop, self.local_hostname)
+        self.stats_mgr.start()
 
         # Set up a pg_notify consumer for allowing web nodes to "provision" and "deprovision" themselves gracefully.
         database_conf = settings.DATABASES['default']
@@ -258,11 +300,10 @@ class WebSocketRelayManager(object):
             **database_conf.get("OPTIONS", {}),
         )
         await async_conn.set_autocommit(True)
-        event_loop.create_task(self.pg_consumer(async_conn))
+        event_loop.create_task(self.on_ws_heartbeat(async_conn))
 
         # Establishes a websocket connection to /websocket/relay on all API servers
         while True:
-            # logger.info("Current known hosts: {}".format(self.known_hosts))
             future_remote_hosts = self.known_hosts.keys()
             current_remote_hosts = self.relay_connections.keys()
             deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts)
@@ -286,18 +327,13 @@ class WebSocketRelayManager(object):
 
             if deleted_remote_hosts:
                 logger.info(f"Removing {deleted_remote_hosts} from websocket broadcast list")
+                await asyncio.gather(self.cleanup_offline_host(h) for h in deleted_remote_hosts)
 
             if new_remote_hosts:
                 logger.info(f"Adding {new_remote_hosts} to websocket broadcast list")
 
-            for h in deleted_remote_hosts:
-                self.relay_connections[h].cancel()
-                del self.relay_connections[h]
-                del self.known_hosts[h]
-                stats_mgr.delete_remote_host_stats(h)
-
             for h in new_remote_hosts:
-                stats = stats_mgr.new_remote_host_stats(h)
+                stats = self.stats_mgr.new_remote_host_stats(h)
                 relay_connection = WebsocketRelayConnection(name=self.local_hostname, stats=stats, remote_host=self.known_hosts[h])
                 relay_connection.start()
                 self.relay_connections[h] = relay_connection

awx/playbooks/action_plugins/insights.py

@@ -58,7 +58,7 @@ class ActionModule(ActionBase):
 
             if res.status_code != 200:
                 result['failed'] = True
-                result['msg'] = 'Expected {} to return a status code of 200 but returned status ' 'code "{}" instead with content "{}".'.format(
+                result['msg'] = 'Expected {} to return a status code of 200 but returned status code "{}" instead with content "{}".'.format(
                     url, res.status_code, res.content
                 )
                 return result
@@ -87,7 +87,7 @@ class ActionModule(ActionBase):
                     continue
                 elif res.status_code != 200:
                     result['failed'] = True
-                    result['msg'] = 'Expected {} to return a status code of 200 but returned status ' 'code "{}" instead with content "{}".'.format(
+                    result['msg'] = 'Expected {} to return a status code of 200 but returned status code "{}" instead with content "{}".'.format(
                         playbook_url, res.status_code, res.content
                     )
                     return result

awx/settings/defaults.py

@@ -114,9 +114,6 @@ JOBOUTPUT_ROOT = '/var/lib/awx/job_status/'
 # Absolute filesystem path to the directory to store logs
 LOG_ROOT = '/var/log/tower/'
 
-# The heartbeat file for the scheduler
-SCHEDULE_METADATA_LOCATION = os.path.join(BASE_DIR, '.tower_cycle')
-
 # Django gettext files path: locale/<lang-code>/LC_MESSAGES/django.po, django.mo
 LOCALE_PATHS = (os.path.join(BASE_DIR, 'locale'),)
 
@@ -395,6 +392,7 @@ AUTHENTICATION_BACKENDS = (
 OAUTH2_PROVIDER_APPLICATION_MODEL = 'main.OAuth2Application'
 OAUTH2_PROVIDER_ACCESS_TOKEN_MODEL = 'main.OAuth2AccessToken'
 OAUTH2_PROVIDER_REFRESH_TOKEN_MODEL = 'oauth2_provider.RefreshToken'
+OAUTH2_PROVIDER_ID_TOKEN_MODEL = "oauth2_provider.IDToken"
 
 OAUTH2_PROVIDER = {'ACCESS_TOKEN_EXPIRE_SECONDS': 31536000000, 'AUTHORIZATION_CODE_EXPIRE_SECONDS': 600, 'REFRESH_TOKEN_EXPIRE_SECONDS': 2628000}
 ALLOW_OAUTH2_FOR_EXTERNAL_USERS = False
@@ -421,6 +419,7 @@ TACACSPLUS_PORT = 49
 TACACSPLUS_SECRET = ''
 TACACSPLUS_SESSION_TIMEOUT = 5
 TACACSPLUS_AUTH_PROTOCOL = 'ascii'
+TACACSPLUS_REM_ADDR = False
 
 # Enable / Disable HTTP Basic Authentication used in the API browser
 # Note: Session limits are not enforced when using HTTP Basic Authentication.
@@ -787,24 +786,18 @@ INSIGHTS_AGENT_MIME = 'application/example'
 INSIGHTS_SYSTEM_ID_FILE = '/etc/redhat-access-insights/machine-id'
 INSIGHTS_CERT_PATH = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"
 
-TOWER_SETTINGS_MANIFEST = {}
-
 # Settings related to external logger configuration
 LOG_AGGREGATOR_ENABLED = False
 LOG_AGGREGATOR_TCP_TIMEOUT = 5
 LOG_AGGREGATOR_VERIFY_CERT = True
 LOG_AGGREGATOR_LEVEL = 'INFO'
-LOG_AGGREGATOR_MAX_DISK_USAGE_GB = 1
+LOG_AGGREGATOR_MAX_DISK_USAGE_GB = 1  # Main queue
+LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB = 1  # Action queue
 LOG_AGGREGATOR_MAX_DISK_USAGE_PATH = '/var/lib/awx'
 LOG_AGGREGATOR_RSYSLOGD_DEBUG = False
 LOG_AGGREGATOR_RSYSLOGD_ERROR_LOG_FILE = '/var/log/tower/rsyslog.err'
 API_400_ERROR_LOG_FORMAT = 'status {status_code} received by user {user_name} attempting to access {url_path} from {remote_addr}'
 
-# The number of retry attempts for websocket session establishment
-# If you're encountering issues establishing websockets in a cluster,
-# raising this value can help
-CHANNEL_LAYER_RECEIVE_MAX_RETRY = 10
-
 ASGI_APPLICATION = "awx.main.routing.application"
 
 CHANNEL_LAYERS = {
@@ -860,17 +853,17 @@ LOGGING = {
         'awx.conf': {'handlers': ['null'], 'level': 'WARNING'},
         'awx.conf.settings': {'handlers': ['null'], 'level': 'WARNING'},
         'awx.main': {'handlers': ['null']},
-        'awx.main.commands.run_callback_receiver': {'handlers': ['callback_receiver']},  # level handled by dynamic_level_filter
+        'awx.main.commands.run_callback_receiver': {'handlers': ['callback_receiver'], 'level': 'INFO'},  # very noisey debug-level logs
         'awx.main.dispatch': {'handlers': ['dispatcher']},
         'awx.main.consumers': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'INFO'},
         'awx.main.rsyslog_configurer': {'handlers': ['rsyslog_configurer']},
         'awx.main.cache_clear': {'handlers': ['cache_clear']},
-        'awx.main.heartbeet': {'handlers': ['heartbeet']},
+        'awx.main.ws_heartbeat': {'handlers': ['ws_heartbeat']},
         'awx.main.wsrelay': {'handlers': ['wsrelay']},
         'awx.main.commands.inventory_import': {'handlers': ['inventory_import'], 'propagate': False},
-        'awx.main.tasks': {'handlers': ['task_system', 'external_logger'], 'propagate': False},
-        'awx.main.analytics': {'handlers': ['task_system', 'external_logger'], 'level': 'INFO', 'propagate': False},
-        'awx.main.scheduler': {'handlers': ['task_system', 'external_logger'], 'propagate': False},
+        'awx.main.tasks': {'handlers': ['task_system', 'external_logger', 'console'], 'propagate': False},
+        'awx.main.analytics': {'handlers': ['task_system', 'external_logger', 'console'], 'level': 'INFO', 'propagate': False},
+        'awx.main.scheduler': {'handlers': ['task_system', 'external_logger', 'console'], 'propagate': False},
         'awx.main.access': {'level': 'INFO'},  # very verbose debug-level logs
         'awx.main.signals': {'level': 'INFO'},  # very verbose debug-level logs
         'awx.api.permissions': {'level': 'INFO'},  # very verbose debug-level logs
@@ -899,7 +892,7 @@ handler_config = {
     'job_lifecycle': {'filename': 'job_lifecycle.log', 'formatter': 'job_lifecycle'},
     'rsyslog_configurer': {'filename': 'rsyslog_configurer.log'},
     'cache_clear': {'filename': 'cache_clear.log'},
-    'heartbeet': {'filename': 'heartbeet.log'},
+    'ws_heartbeat': {'filename': 'ws_heartbeat.log'},
 }
 
 # If running on a VM, we log to files. When running in a container, we log to stdout.
@@ -954,7 +947,8 @@ AWX_CLEANUP_PATHS = True
 # Allow ansible-runner to store env folder (may contain sensitive information)
 AWX_RUNNER_OMIT_ENV_FILES = True
 
-# Allow ansible-runner to save ansible output (may cause performance issues)
+# Allow ansible-runner to save ansible output
+# (changing to False may cause performance issues)
 AWX_RUNNER_SUPPRESS_OUTPUT_FILE = True
 
 # https://github.com/ansible/ansible-runner/pull/1191/files

awx/settings/development.py

@@ -29,8 +29,6 @@ SHELL_PLUS_PRINT_SQL = False
 # show colored logs in the dev environment
 # to disable this, set `COLOR_LOGS = False` in awx/settings/local_settings.py
 LOGGING['handlers']['console']['()'] = 'awx.main.utils.handlers.ColorHandler'  # noqa
-# task system does not propagate to AWX, so color log these too
-LOGGING['handlers']['task_system'] = LOGGING['handlers']['console'].copy()  # noqa
 COLOR_LOGS = True
 
 ALLOWED_HOSTS = ['*']
@@ -51,7 +49,7 @@ INSIGHTS_TRACKING_STATE = False
 
 # debug toolbar and swagger assume that requirements/requirements_dev.txt are installed
 
-INSTALLED_APPS += ['rest_framework_swagger', 'debug_toolbar']  # NOQA
+INSTALLED_APPS += ['drf_yasg', 'debug_toolbar']  # NOQA
 
 MIDDLEWARE = ['debug_toolbar.middleware.DebugToolbarMiddleware'] + MIDDLEWARE  # NOQA
 
@@ -61,8 +59,9 @@ DEBUG_TOOLBAR_CONFIG = {'ENABLE_STACKTRACES': True}
 SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
 INSTALL_UUID = '00000000-0000-0000-0000-000000000000'
 
-BASE_VENV_PATH = "/var/lib/awx/venv/"
-AWX_VENV_PATH = os.path.join(BASE_VENV_PATH, "awx")
+# Ansible base virtualenv paths and enablement
+# only used for deprecated fields and management commands for them
+BASE_VENV_PATH = os.path.realpath("/var/lib/awx/venv")
 
 CLUSTER_HOST_ID = socket.gethostname()
 

awx/settings/production.py

@@ -30,15 +30,10 @@ SECRET_KEY = None
 # See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = []
 
-# The heartbeat file for the scheduler
-SCHEDULE_METADATA_LOCATION = '/var/lib/awx/.tower_cycle'
-
 # Ansible base virtualenv paths and enablement
+# only used for deprecated fields and management commands for them
 BASE_VENV_PATH = os.path.realpath("/var/lib/awx/venv")
 
-# Base virtualenv paths and enablement
-AWX_VENV_PATH = os.path.join(BASE_VENV_PATH, "awx")
-
 # Very important that this is editable (not read_only) in the API
 AWX_ISOLATION_SHOW_PATHS = [
     '/etc/pki/ca-trust:/etc/pki/ca-trust:O',

awx/sso/backends.py

@@ -224,12 +224,18 @@ class TACACSPlusBackend(object):
             return None
         try:
             # Upstream TACACS+ client does not accept non-string, so convert if needed.
-            auth = tacacs_plus.TACACSClient(
+            tacacs_client = tacacs_plus.TACACSClient(
                 django_settings.TACACSPLUS_HOST,
                 django_settings.TACACSPLUS_PORT,
                 django_settings.TACACSPLUS_SECRET,
                 timeout=django_settings.TACACSPLUS_SESSION_TIMEOUT,
-            ).authenticate(username, password, authen_type=tacacs_plus.TAC_PLUS_AUTHEN_TYPES[django_settings.TACACSPLUS_AUTH_PROTOCOL])
+            )
+            auth_kwargs = {'authen_type': tacacs_plus.TAC_PLUS_AUTHEN_TYPES[django_settings.TACACSPLUS_AUTH_PROTOCOL]}
+            if django_settings.TACACSPLUS_AUTH_PROTOCOL:
+                client_ip = self._get_client_ip(request)
+                if client_ip:
+                    auth_kwargs['rem_addr'] = client_ip
+            auth = tacacs_client.authenticate(username, password, **auth_kwargs)
         except Exception as e:
             logger.exception("TACACS+ Authentication Error: %s" % str(e))
             return None
@@ -244,6 +250,17 @@ class TACACSPlusBackend(object):
         except User.DoesNotExist:
             return None
 
+    def _get_client_ip(self, request):
+        if not request or not hasattr(request, 'META'):
+            return None
+
+        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
+        if x_forwarded_for:
+            ip = x_forwarded_for.split(',')[0]
+        else:
+            ip = request.META.get('REMOTE_ADDR')
+        return ip
+
 
 class TowerSAMLIdentityProvider(BaseSAMLIdentityProvider):
     """
@@ -269,7 +286,7 @@ class TowerSAMLIdentityProvider(BaseSAMLIdentityProvider):
             value = value[0]
         if conf_key in ('attr_first_name', 'attr_last_name', 'attr_username', 'attr_email') and value is None:
             logger.warning(
-                "Could not map user detail '%s' from SAML attribute '%s'; " "update SOCIAL_AUTH_SAML_ENABLED_IDPS['%s']['%s'] with the correct SAML attribute.",
+                "Could not map user detail '%s' from SAML attribute '%s'; update SOCIAL_AUTH_SAML_ENABLED_IDPS['%s']['%s'] with the correct SAML attribute.",
                 conf_key[5:],
                 key,
                 self.name,

awx/sso/conf.py

@@ -100,7 +100,7 @@ register(
     'AUTHENTICATION_BACKENDS',
     field_class=AuthenticationBackendsField,
     label=_('Authentication Backends'),
-    help_text=_('List of authentication backends that are enabled based on ' 'license features and other authentication settings.'),
+    help_text=_('List of authentication backends that are enabled based on license features and other authentication settings.'),
     read_only=True,
     depends_on=AuthenticationBackendsField.get_all_required_settings(),
     category=_('Authentication'),
@@ -360,7 +360,7 @@ def _register_ldap(append=None):
         default=None,
         label=_('LDAP Deny Group'),
         help_text=_(
-            'Group DN denied from login. If specified, user will not be ' 'allowed to login if a member of this group.  Only one deny group ' 'is supported.'
+            'Group DN denied from login. If specified, user will not be allowed to login if a member of this group.  Only one deny group is supported.'
         ),
         category=_('LDAP'),
         category_slug='ldap',
@@ -426,7 +426,7 @@ def _register_ldap(append=None):
         field_class=LDAPTeamMapField,
         default={},
         label=_('LDAP Team Map'),
-        help_text=_('Mapping between team members (users) and LDAP groups. Configuration' ' details are available in the documentation.'),
+        help_text=_('Mapping between team members (users) and LDAP groups. Configuration details are available in the documentation.'),
         category=_('LDAP'),
         category_slug='ldap',
         placeholder=collections.OrderedDict(
@@ -461,7 +461,7 @@ register(
     allow_blank=True,
     default='',
     label=_('RADIUS Server'),
-    help_text=_('Hostname/IP of RADIUS server. RADIUS authentication is ' 'disabled if this setting is empty.'),
+    help_text=_('Hostname/IP of RADIUS server. RADIUS authentication is disabled if this setting is empty.'),
     category=_('RADIUS'),
     category_slug='radius',
     placeholder='radius.example.com',
@@ -554,6 +554,16 @@ register(
     category_slug='tacacsplus',
 )
 
+register(
+    'TACACSPLUS_REM_ADDR',
+    field_class=fields.BooleanField,
+    default=True,
+    label=_('TACACS+ client address sending enabled'),
+    help_text=_('Enable the client address sending by TACACS+ client.'),
+    category=_('TACACS+'),
+    category_slug='tacacsplus',
+)
+
 ###############################################################################
 # GOOGLE OAUTH2 AUTHENTICATION SETTINGS
 ###############################################################################
@@ -564,9 +574,7 @@ register(
     read_only=True,
     default=SocialAuthCallbackURL('google-oauth2'),
     label=_('Google OAuth2 Callback URL'),
-    help_text=_(
-        'Provide this URL as the callback URL for your application as part ' 'of your registration process. Refer to the ' 'documentation for more detail.'
-    ),
+    help_text=_('Provide this URL as the callback URL for your application as part of your registration process. Refer to the documentation for more detail.'),
     category=_('Google OAuth2'),
     category_slug='google-oauth2',
     depends_on=['TOWER_URL_BASE'],
@@ -602,7 +610,7 @@ register(
     field_class=fields.StringListField,
     default=[],
     label=_('Google OAuth2 Allowed Domains'),
-    help_text=_('Update this setting to restrict the domains who are allowed to ' 'login using Google OAuth2.'),
+    help_text=_('Update this setting to restrict the domains who are allowed to login using Google OAuth2.'),
     category=_('Google OAuth2'),
     category_slug='google-oauth2',
     placeholder=['example.com'],
@@ -658,9 +666,7 @@ register(
     read_only=True,
     default=SocialAuthCallbackURL('github'),
     label=_('GitHub OAuth2 Callback URL'),
-    help_text=_(
-        'Provide this URL as the callback URL for your application as part ' 'of your registration process. Refer to the ' 'documentation for more detail.'
-    ),
+    help_text=_('Provide this URL as the callback URL for your application as part of your registration process. Refer to the documentation for more detail.'),
     category=_('GitHub OAuth2'),
     category_slug='github',
     depends_on=['TOWER_URL_BASE'],
@@ -723,9 +729,7 @@ register(
     read_only=True,
     default=SocialAuthCallbackURL('github-org'),
     label=_('GitHub Organization OAuth2 Callback URL'),
-    help_text=_(
-        'Provide this URL as the callback URL for your application as part ' 'of your registration process. Refer to the ' 'documentation for more detail.'
-    ),
+    help_text=_('Provide this URL as the callback URL for your application as part of your registration process. Refer to the documentation for more detail.'),
     category=_('GitHub Organization OAuth2'),
     category_slug='github-org',
     depends_on=['TOWER_URL_BASE'],
@@ -760,7 +764,7 @@ register(
     allow_blank=True,
     default='',
     label=_('GitHub Organization Name'),
-    help_text=_('The name of your GitHub organization, as used in your ' 'organization\'s URL: https://github.com/<yourorg>/.'),
+    help_text=_('The name of your GitHub organization, as used in your organization\'s URL: https://github.com/<yourorg>/.'),
     category=_('GitHub Organization OAuth2'),
     category_slug='github-org',
 )
@@ -839,7 +843,7 @@ register(
     allow_blank=True,
     default='',
     label=_('GitHub Team ID'),
-    help_text=_('Find the numeric team ID using the Github API: ' 'http://fabian-kostadinov.github.io/2015/01/16/how-to-find-a-github-team-id/.'),
+    help_text=_('Find the numeric team ID using the Github API: http://fabian-kostadinov.github.io/2015/01/16/how-to-find-a-github-team-id/.'),
     category=_('GitHub Team OAuth2'),
     category_slug='github-team',
 )
@@ -878,9 +882,7 @@ register(
     read_only=True,
     default=SocialAuthCallbackURL('github-enterprise'),
     label=_('GitHub Enterprise OAuth2 Callback URL'),
-    help_text=_(
-        'Provide this URL as the callback URL for your application as part ' 'of your registration process. Refer to the ' 'documentation for more detail.'
-    ),
+    help_text=_('Provide this URL as the callback URL for your application as part of your registration process. Refer to the documentation for more detail.'),
     category=_('GitHub Enterprise OAuth2'),
     category_slug='github-enterprise',
     depends_on=['TOWER_URL_BASE'],
@@ -892,7 +894,7 @@ register(
     allow_blank=True,
     default='',
     label=_('GitHub Enterprise URL'),
-    help_text=_('The URL for your Github Enterprise instance, e.g.: http(s)://hostname/. Refer to Github Enterprise ' 'documentation for more details.'),
+    help_text=_('The URL for your Github Enterprise instance, e.g.: http(s)://hostname/. Refer to Github Enterprise documentation for more details.'),
     category=_('GitHub Enterprise OAuth2'),
     category_slug='github-enterprise',
 )
@@ -904,7 +906,7 @@ register(
     default='',
     label=_('GitHub Enterprise API URL'),
     help_text=_(
-        'The API URL for your GitHub Enterprise instance, e.g.: http(s)://hostname/api/v3/. Refer to Github ' 'Enterprise documentation for more details.'
+        'The API URL for your GitHub Enterprise instance, e.g.: http(s)://hostname/api/v3/. Refer to Github Enterprise documentation for more details.'
     ),
     category=_('GitHub Enterprise OAuth2'),
     category_slug='github-enterprise',
@@ -967,9 +969,7 @@ register(
     read_only=True,
     default=SocialAuthCallbackURL('github-enterprise-org'),
     label=_('GitHub Enterprise Organization OAuth2 Callback URL'),
-    help_text=_(
-        'Provide this URL as the callback URL for your application as part ' 'of your registration process. Refer to the ' 'documentation for more detail.'
-    ),
+    help_text=_('Provide this URL as the callback URL for your application as part of your registration process. Refer to the documentation for more detail.'),
     category=_('GitHub Enterprise Organization OAuth2'),
     category_slug='github-enterprise-org',
     depends_on=['TOWER_URL_BASE'],
@@ -981,7 +981,7 @@ register(
     allow_blank=True,
     default='',
     label=_('GitHub Enterprise Organization URL'),
-    help_text=_('The URL for your Github Enterprise instance, e.g.: http(s)://hostname/. Refer to Github Enterprise ' 'documentation for more details.'),
+    help_text=_('The URL for your Github Enterprise instance, e.g.: http(s)://hostname/. Refer to Github Enterprise documentation for more details.'),
     category=_('GitHub Enterprise OAuth2'),
     category_slug='github-enterprise-org',
 )
@@ -993,7 +993,7 @@ register(
     default='',
     label=_('GitHub Enterprise Organization API URL'),
     help_text=_(
-        'The API URL for your GitHub Enterprise instance, e.g.: http(s)://hostname/api/v3/. Refer to Github ' 'Enterprise documentation for more details.'
+        'The API URL for your GitHub Enterprise instance, e.g.: http(s)://hostname/api/v3/. Refer to Github Enterprise documentation for more details.'
     ),
     category=_('GitHub Enterprise OAuth2'),
     category_slug='github-enterprise-org',
@@ -1028,7 +1028,7 @@ register(
     allow_blank=True,
     default='',
     label=_('GitHub Enterprise Organization Name'),
-    help_text=_('The name of your GitHub Enterprise organization, as used in your ' 'organization\'s URL: https://github.com/<yourorg>/.'),
+    help_text=_('The name of your GitHub Enterprise organization, as used in your organization\'s URL: https://github.com/<yourorg>/.'),
     category=_('GitHub Enterprise Organization OAuth2'),
     category_slug='github-enterprise-org',
 )
@@ -1084,7 +1084,7 @@ register(
     allow_blank=True,
     default='',
     label=_('GitHub Enterprise Team URL'),
-    help_text=_('The URL for your Github Enterprise instance, e.g.: http(s)://hostname/. Refer to Github Enterprise ' 'documentation for more details.'),
+    help_text=_('The URL for your Github Enterprise instance, e.g.: http(s)://hostname/. Refer to Github Enterprise documentation for more details.'),
     category=_('GitHub Enterprise OAuth2'),
     category_slug='github-enterprise-team',
 )
@@ -1096,7 +1096,7 @@ register(
     default='',
     label=_('GitHub Enterprise Team API URL'),
     help_text=_(
-        'The API URL for your GitHub Enterprise instance, e.g.: http(s)://hostname/api/v3/. Refer to Github ' 'Enterprise documentation for more details.'
+        'The API URL for your GitHub Enterprise instance, e.g.: http(s)://hostname/api/v3/. Refer to Github Enterprise documentation for more details.'
     ),
     category=_('GitHub Enterprise OAuth2'),
     category_slug='github-enterprise-team',
@@ -1131,7 +1131,7 @@ register(
     allow_blank=True,
     default='',
     label=_('GitHub Enterprise Team ID'),
-    help_text=_('Find the numeric team ID using the Github Enterprise API: ' 'http://fabian-kostadinov.github.io/2015/01/16/how-to-find-a-github-team-id/.'),
+    help_text=_('Find the numeric team ID using the Github Enterprise API: http://fabian-kostadinov.github.io/2015/01/16/how-to-find-a-github-team-id/.'),
     category=_('GitHub Enterprise Team OAuth2'),
     category_slug='github-enterprise-team',
 )
@@ -1170,9 +1170,7 @@ register(
     read_only=True,
     default=SocialAuthCallbackURL('azuread-oauth2'),
     label=_('Azure AD OAuth2 Callback URL'),
-    help_text=_(
-        'Provide this URL as the callback URL for your application as part' ' of your registration process. Refer to the' ' documentation for more detail. '
-    ),
+    help_text=_('Provide this URL as the callback URL for your application as part of your registration process. Refer to the documentation for more detail. '),
     category=_('Azure AD OAuth2'),
     category_slug='azuread-oauth2',
     depends_on=['TOWER_URL_BASE'],
@@ -1291,7 +1289,7 @@ register(
     field_class=fields.BooleanField,
     default=True,
     label=_('Automatically Create Organizations and Teams on SAML Login'),
-    help_text=_('When enabled (the default), mapped Organizations and Teams ' 'will be created automatically on successful SAML login.'),
+    help_text=_('When enabled (the default), mapped Organizations and Teams will be created automatically on successful SAML login.'),
     category=_('SAML'),
     category_slug='saml',
 )
@@ -1318,7 +1316,7 @@ register(
     read_only=True,
     default=get_saml_metadata_url,
     label=_('SAML Service Provider Metadata URL'),
-    help_text=_('If your identity provider (IdP) allows uploading an XML ' 'metadata file, you can download one from this URL.'),
+    help_text=_('If your identity provider (IdP) allows uploading an XML metadata file, you can download one from this URL.'),
     category=_('SAML'),
     category_slug='saml',
 )
@@ -1346,7 +1344,7 @@ register(
     required=True,
     validators=[validate_certificate],
     label=_('SAML Service Provider Public Certificate'),
-    help_text=_('Create a keypair to use as a service provider (SP) ' 'and include the certificate content here.'),
+    help_text=_('Create a keypair to use as a service provider (SP) and include the certificate content here.'),
     category=_('SAML'),
     category_slug='saml',
 )
@@ -1358,7 +1356,7 @@ register(
     required=True,
     validators=[validate_private_key],
     label=_('SAML Service Provider Private Key'),
-    help_text=_('Create a keypair to use as a service provider (SP) ' 'and include the private key content here.'),
+    help_text=_('Create a keypair to use as a service provider (SP) and include the private key content here.'),
     category=_('SAML'),
     category_slug='saml',
     encrypted=True,
@@ -1369,7 +1367,7 @@ register(
     field_class=SAMLOrgInfoField,
     required=True,
     label=_('SAML Service Provider Organization Info'),
-    help_text=_('Provide the URL, display name, and the name of your app. Refer to' ' the documentation for example syntax.'),
+    help_text=_('Provide the URL, display name, and the name of your app. Refer to the documentation for example syntax.'),
     category=_('SAML'),
     category_slug='saml',
     placeholder=collections.OrderedDict(
@@ -1383,7 +1381,7 @@ register(
     allow_blank=True,
     required=True,
     label=_('SAML Service Provider Technical Contact'),
-    help_text=_('Provide the name and email address of the technical contact for' ' your service provider. Refer to the documentation' ' for example syntax.'),
+    help_text=_('Provide the name and email address of the technical contact for your service provider. Refer to the documentation for example syntax.'),
     category=_('SAML'),
     category_slug='saml',
     placeholder=collections.OrderedDict([('givenName', 'Technical Contact'), ('emailAddress', 'techsup@example.com')]),
@@ -1395,7 +1393,7 @@ register(
     allow_blank=True,
     required=True,
     label=_('SAML Service Provider Support Contact'),
-    help_text=_('Provide the name and email address of the support contact for your' ' service provider. Refer to the documentation for' ' example syntax.'),
+    help_text=_('Provide the name and email address of the support contact for your service provider. Refer to the documentation for example syntax.'),
     category=_('SAML'),
     category_slug='saml',
     placeholder=collections.OrderedDict([('givenName', 'Support Contact'), ('emailAddress', 'support@example.com')]),
@@ -1457,9 +1455,7 @@ register(
     allow_null=True,
     default={'requestedAuthnContext': False},
     label=_('SAML Security Config'),
-    help_text=_(
-        'A dict of key value pairs that are passed to the underlying' ' python-saml security setting' ' https://github.com/onelogin/python-saml#settings'
-    ),
+    help_text=_('A dict of key value pairs that are passed to the underlying python-saml security setting https://github.com/onelogin/python-saml#settings'),
     category=_('SAML'),
     category_slug='saml',
     placeholder=collections.OrderedDict(
@@ -1491,7 +1487,7 @@ register(
     allow_null=True,
     default=None,
     label=_('SAML Service Provider extra configuration data'),
-    help_text=_('A dict of key value pairs to be passed to the underlying' ' python-saml Service Provider configuration setting.'),
+    help_text=_('A dict of key value pairs to be passed to the underlying python-saml Service Provider configuration setting.'),
     category=_('SAML'),
     category_slug='saml',
     placeholder=collections.OrderedDict(),