Merge branch 'ansible:devel' into hashicorp-vault-kubernetes-auth

awx/main/__init__.py

@@ -1,4 +1,2 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
-
-default_app_config = 'awx.main.apps.MainConfig'

awx/main/access.py

@@ -11,7 +11,7 @@ from functools import reduce
 from django.conf import settings
 from django.db.models import Q, Prefetch
 from django.contrib.auth.models import User
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import ObjectDoesNotExist
 
 # Django REST Framework
@@ -465,7 +465,7 @@ class BaseAccess(object):
             if display_method == 'schedule':
                 user_capabilities['schedule'] = user_capabilities['start']
                 continue
-            elif display_method == 'delete' and not isinstance(obj, (User, UnifiedJob, CredentialInputSource, ExecutionEnvironment)):
+            elif display_method == 'delete' and not isinstance(obj, (User, UnifiedJob, CredentialInputSource, ExecutionEnvironment, InstanceGroup)):
                 user_capabilities['delete'] = user_capabilities['edit']
                 continue
             elif display_method == 'copy' and isinstance(obj, (Group, Host)):
@@ -575,6 +575,11 @@ class InstanceGroupAccess(BaseAccess):
     def can_change(self, obj, data):
         return self.user.is_superuser
 
+    def can_delete(self, obj):
+        if obj.name in [settings.DEFAULT_EXECUTION_QUEUE_NAME, settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME]:
+            return False
+        return self.user.is_superuser
+
 
 class UserAccess(BaseAccess):
     """

awx/main/analytics/broadcast_websocket.py

@@ -89,7 +89,7 @@ class BroadcastWebsocketStatsManager:
 
                 await asyncio.sleep(settings.BROADCAST_WEBSOCKET_STATS_POLL_RATE_SECONDS)
         except Exception as e:
-            logger.warn(e)
+            logger.warning(e)
             await asyncio.sleep(settings.BROADCAST_WEBSOCKET_STATS_POLL_RATE_SECONDS)
             self.start()
 

awx/main/analytics/collectors.py

@@ -10,7 +10,7 @@ from django.db.models import Count
 from django.conf import settings
 from django.contrib.sessions.models import Session
 from django.utils.timezone import now, timedelta
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from psycopg2.errors import UntranslatableCharacter
 
@@ -337,6 +337,7 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
                           {tbl}.parent_uuid,
                           {tbl}.event,
                           task_action,
+                          resolved_action,
                           -- '-' operator listed here:
                           -- https://www.postgresql.org/docs/12/functions-json.html
                           -- note that operator is only supported by jsonb objects
@@ -356,7 +357,7 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
                           x.duration AS duration,
                           x.res->'warnings' AS warnings,
                           x.res->'deprecations' AS deprecations
-                          FROM {tbl}, jsonb_to_record({event_data}) AS x("res" json, "duration" text, "task_action" text, "start" text, "end" text)
+                          FROM {tbl}, jsonb_to_record({event_data}) AS x("res" json, "duration" text, "task_action" text, "resolved_action" text, "start" text, "end" text)
                           WHERE ({tbl}.{where_column} > '{since.isoformat()}' AND {tbl}.{where_column} <= '{until.isoformat()}')) TO STDOUT WITH CSV HEADER'''
         return query
 
@@ -366,23 +367,24 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
         return _copy_table(table='events', query=query(f"replace({tbl}.event_data::text, '\\u0000', '')::jsonb"), path=full_path)
 
 
-@register('events_table', '1.3', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
+@register('events_table', '1.4', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
 def events_table_unpartitioned(since, full_path, until, **kwargs):
     return _events_table(since, full_path, until, '_unpartitioned_main_jobevent', 'created', **kwargs)
 
 
-@register('events_table', '1.3', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
+@register('events_table', '1.4', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
 def events_table_partitioned_modified(since, full_path, until, **kwargs):
     return _events_table(since, full_path, until, 'main_jobevent', 'modified', project_job_created=True, **kwargs)
 
 
-@register('unified_jobs_table', '1.2', format='csv', description=_('Data on jobs run'), expensive=four_hour_slicing)
+@register('unified_jobs_table', '1.3', format='csv', description=_('Data on jobs run'), expensive=four_hour_slicing)
 def unified_jobs_table(since, full_path, until, **kwargs):
     unified_job_query = '''COPY (SELECT main_unifiedjob.id,
                                  main_unifiedjob.polymorphic_ctype_id,
                                  django_content_type.model,
                                  main_unifiedjob.organization_id,
                                  main_organization.name as organization_name,
+                                 main_executionenvironment.image as execution_environment_image,
                                  main_job.inventory_id,
                                  main_inventory.name as inventory_name,
                                  main_unifiedjob.created,
@@ -407,6 +409,7 @@ def unified_jobs_table(since, full_path, until, **kwargs):
                                  LEFT JOIN main_job ON main_unifiedjob.id = main_job.unifiedjob_ptr_id
                                  LEFT JOIN main_inventory ON main_job.inventory_id = main_inventory.id
                                  LEFT JOIN main_organization ON main_organization.id = main_unifiedjob.organization_id
+                                 LEFT JOIN main_executionenvironment ON main_executionenvironment.id = main_unifiedjob.execution_environment_id
                                  WHERE ((main_unifiedjob.created > '{0}' AND main_unifiedjob.created <= '{1}')
                                        OR (main_unifiedjob.finished > '{0}' AND main_unifiedjob.finished <= '{1}'))
                                        AND main_unifiedjob.launch_type != 'sync'
@@ -417,11 +420,12 @@ def unified_jobs_table(since, full_path, until, **kwargs):
     return _copy_table(table='unified_jobs', query=unified_job_query, path=full_path)
 
 
-@register('unified_job_template_table', '1.0', format='csv', description=_('Data on job templates'))
+@register('unified_job_template_table', '1.1', format='csv', description=_('Data on job templates'))
 def unified_job_template_table(since, full_path, **kwargs):
     unified_job_template_query = '''COPY (SELECT main_unifiedjobtemplate.id,
                                  main_unifiedjobtemplate.polymorphic_ctype_id,
                                  django_content_type.model,
+                                 main_executionenvironment.image as execution_environment_image,
                                  main_unifiedjobtemplate.created,
                                  main_unifiedjobtemplate.modified,
                                  main_unifiedjobtemplate.created_by_id,
@@ -434,7 +438,8 @@ def unified_job_template_table(since, full_path, **kwargs):
                                  main_unifiedjobtemplate.next_job_run,
                                  main_unifiedjobtemplate.next_schedule_id,
                                  main_unifiedjobtemplate.status
-                                 FROM main_unifiedjobtemplate, django_content_type
+                                 FROM main_unifiedjobtemplate
+                                 LEFT JOIN main_executionenvironment ON main_executionenvironment.id = main_unifiedjobtemplate.execution_environment_id, django_content_type
                                  WHERE main_unifiedjobtemplate.polymorphic_ctype_id = django_content_type.id
                                  ORDER BY main_unifiedjobtemplate.id ASC) TO STDOUT WITH CSV HEADER'''
     return _copy_table(table='unified_job_template', query=unified_job_template_query, path=full_path)

awx/main/apps.py

@@ -1,5 +1,5 @@
 from django.apps import AppConfig
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 
 class MainConfig(AppConfig):

awx/main/conf.py

@@ -2,7 +2,7 @@
 import logging
 
 # Django
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 # Django REST Framework
 from rest_framework import serializers
@@ -334,6 +334,19 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'AWX_MOUNT_ISOLATED_PATHS_ON_K8S',
+    field_class=fields.BooleanField,
+    default=False,
+    label=_('Expose host paths for Container Groups'),
+    help_text=_(
+        'Expose paths via hostPath for the Pods created by a Container Group. '
+        'HostPath volumes present many security risks, and it is a best practice to avoid the use of HostPaths when possible. '
+    ),
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
 register(
     'GALAXY_IGNORE_CERTS',
     field_class=fields.BooleanField,

awx/main/constants.py

@@ -3,7 +3,7 @@
 
 import re
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 __all__ = [
     'CLOUD_PROVIDERS',
@@ -88,7 +88,10 @@ JOB_FOLDER_PREFIX = 'awx_%s_'
 
 # :z option tells Podman that two containers share the volume content with r/w
 # :O option tells Podman to mount the directory from the host as a temporary storage using the overlay file system.
+# :ro or :rw option to mount a volume in read-only or read-write mode, respectively. By default, the volumes are mounted read-write.
 # see podman-run manpage for further details
 # /HOST-DIR:/CONTAINER-DIR:OPTIONS
-CONTAINER_VOLUMES_MOUNT_TYPES = ['z', 'O']
+CONTAINER_VOLUMES_MOUNT_TYPES = ['z', 'O', 'ro', 'rw']
 MAX_ISOLATED_PATH_COLON_DELIMITER = 2
+
+SURVEY_TYPE_MAPPING = {'text': str, 'textarea': str, 'password': str, 'multiplechoice': str, 'multiselect': str, 'integer': int, 'float': (float, int)}

awx/main/consumers.py

@@ -65,7 +65,7 @@ class WebsocketSecretAuthHelper:
         nonce_parsed = int(nonce_parsed)
         nonce_diff = now - nonce_parsed
         if abs(nonce_diff) > nonce_tolerance:
-            logger.warn(f"Potential replay attack or machine(s) time out of sync by {nonce_diff} seconds.")
+            logger.warning(f"Potential replay attack or machine(s) time out of sync by {nonce_diff} seconds.")
             raise ValueError(f"Potential replay attack or machine(s) time out of sync by {nonce_diff} seconds.")
 
         return True
@@ -85,7 +85,7 @@ class BroadcastConsumer(AsyncJsonWebsocketConsumer):
         try:
             WebsocketSecretAuthHelper.is_authorized(self.scope)
         except Exception:
-            logger.warn(f"client '{self.channel_name}' failed to authorize against the broadcast endpoint.")
+            logger.warning(f"client '{self.channel_name}' failed to authorize against the broadcast endpoint.")
             await self.close()
             return
 

awx/main/credential_plugins/aim.py

@@ -2,7 +2,7 @@ from .plugin import CredentialPlugin, CertFiles, raise_for_status
 
 from urllib.parse import quote, urlencode, urljoin
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 import requests
 
 aim_inputs = {

awx/main/credential_plugins/azure_kv.py

@@ -1,6 +1,6 @@
 from .plugin import CredentialPlugin
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from azure.keyvault import KeyVaultClient, KeyVaultAuthentication
 from azure.common.credentials import ServicePrincipalCredentials
 from msrestazure import azure_cloud

awx/main/credential_plugins/centrify_vault.py

@@ -1,115 +1,115 @@
-from .plugin import CredentialPlugin, raise_for_status
-from django.utils.translation import ugettext_lazy as _
-from urllib.parse import urljoin
-import requests
-
-pas_inputs = {
-    'fields': [
-        {
-            'id': 'url',
-            'label': _('Centrify Tenant URL'),
-            'type': 'string',
-            'help_text': _('Centrify Tenant URL'),
-            'format': 'url',
-        },
-        {
-            'id': 'client_id',
-            'label': _('Centrify API User'),
-            'type': 'string',
-            'help_text': _('Centrify API User, having necessary permissions as mentioned in support doc'),
-        },
-        {
-            'id': 'client_password',
-            'label': _('Centrify API Password'),
-            'type': 'string',
-            'help_text': _('Password of Centrify API User with necessary permissions'),
-            'secret': True,
-        },
-        {
-            'id': 'oauth_application_id',
-            'label': _('OAuth2 Application ID'),
-            'type': 'string',
-            'help_text': _('Application ID of the configured OAuth2 Client (defaults to \'awx\')'),
-            'default': 'awx',
-        },
-        {
-            'id': 'oauth_scope',
-            'label': _('OAuth2 Scope'),
-            'type': 'string',
-            'help_text': _('Scope of the configured OAuth2 Client (defaults to \'awx\')'),
-            'default': 'awx',
-        },
-    ],
-    'metadata': [
-        {
-            'id': 'account-name',
-            'label': _('Account Name'),
-            'type': 'string',
-            'help_text': _('Local system account or Domain account name enrolled in Centrify Vault. eg. (root or DOMAIN/Administrator)'),
-        },
-        {
-            'id': 'system-name',
-            'label': _('System Name'),
-            'type': 'string',
-            'help_text': _('Machine Name enrolled with in Centrify Portal'),
-        },
-    ],
-    'required': ['url', 'account-name', 'system-name', 'client_id', 'client_password'],
-}
-
-
-# generate bearer token to authenticate with PAS portal, Input : Client ID, Client Secret
-def handle_auth(**kwargs):
-    post_data = {"grant_type": "client_credentials", "scope": kwargs['oauth_scope']}
-    response = requests.post(kwargs['endpoint'], data=post_data, auth=(kwargs['client_id'], kwargs['client_password']), verify=True, timeout=(5, 30))
-    raise_for_status(response)
-    try:
-        return response.json()['access_token']
-    except KeyError:
-        raise RuntimeError('OAuth request to tenant was unsuccessful')
-
-
-# fetch the ID of system with RedRock query, Input : System Name, Account Name
-def get_ID(**kwargs):
-    endpoint = urljoin(kwargs['url'], '/Redrock/query')
-    name = " Name='{0}' and User='{1}'".format(kwargs['system_name'], kwargs['acc_name'])
-    query = 'Select ID from VaultAccount where {0}'.format(name)
-    post_headers = {"Authorization": "Bearer " + kwargs['access_token'], "X-CENTRIFY-NATIVE-CLIENT": "true"}
-    response = requests.post(endpoint, json={'Script': query}, headers=post_headers, verify=True, timeout=(5, 30))
-    raise_for_status(response)
-    try:
-        result_str = response.json()["Result"]["Results"]
-        return result_str[0]["Row"]["ID"]
-    except (IndexError, KeyError):
-        raise RuntimeError("Error Detected!! Check the Inputs")
-
-
-# CheckOut Password from Centrify Vault, Input : ID
-def get_passwd(**kwargs):
-    endpoint = urljoin(kwargs['url'], '/ServerManage/CheckoutPassword')
-    post_headers = {"Authorization": "Bearer " + kwargs['access_token'], "X-CENTRIFY-NATIVE-CLIENT": "true"}
-    response = requests.post(endpoint, json={'ID': kwargs['acc_id']}, headers=post_headers, verify=True, timeout=(5, 30))
-    raise_for_status(response)
-    try:
-        return response.json()["Result"]["Password"]
-    except KeyError:
-        raise RuntimeError("Password Not Found")
-
-
-def centrify_backend(**kwargs):
-    url = kwargs.get('url')
-    acc_name = kwargs.get('account-name')
-    system_name = kwargs.get('system-name')
-    client_id = kwargs.get('client_id')
-    client_password = kwargs.get('client_password')
-    app_id = kwargs.get('oauth_application_id', 'awx')
-    endpoint = urljoin(url, f'/oauth2/token/{app_id}')
-    endpoint = {'endpoint': endpoint, 'client_id': client_id, 'client_password': client_password, 'oauth_scope': kwargs.get('oauth_scope', 'awx')}
-    token = handle_auth(**endpoint)
-    get_id_args = {'system_name': system_name, 'acc_name': acc_name, 'url': url, 'access_token': token}
-    acc_id = get_ID(**get_id_args)
-    get_pwd_args = {'url': url, 'acc_id': acc_id, 'access_token': token}
-    return get_passwd(**get_pwd_args)
-
-
-centrify_plugin = CredentialPlugin('Centrify Vault Credential Provider Lookup', inputs=pas_inputs, backend=centrify_backend)
+from .plugin import CredentialPlugin, raise_for_status
+from django.utils.translation import gettext_lazy as _
+from urllib.parse import urljoin
+import requests
+
+pas_inputs = {
+    'fields': [
+        {
+            'id': 'url',
+            'label': _('Centrify Tenant URL'),
+            'type': 'string',
+            'help_text': _('Centrify Tenant URL'),
+            'format': 'url',
+        },
+        {
+            'id': 'client_id',
+            'label': _('Centrify API User'),
+            'type': 'string',
+            'help_text': _('Centrify API User, having necessary permissions as mentioned in support doc'),
+        },
+        {
+            'id': 'client_password',
+            'label': _('Centrify API Password'),
+            'type': 'string',
+            'help_text': _('Password of Centrify API User with necessary permissions'),
+            'secret': True,
+        },
+        {
+            'id': 'oauth_application_id',
+            'label': _('OAuth2 Application ID'),
+            'type': 'string',
+            'help_text': _('Application ID of the configured OAuth2 Client (defaults to \'awx\')'),
+            'default': 'awx',
+        },
+        {
+            'id': 'oauth_scope',
+            'label': _('OAuth2 Scope'),
+            'type': 'string',
+            'help_text': _('Scope of the configured OAuth2 Client (defaults to \'awx\')'),
+            'default': 'awx',
+        },
+    ],
+    'metadata': [
+        {
+            'id': 'account-name',
+            'label': _('Account Name'),
+            'type': 'string',
+            'help_text': _('Local system account or Domain account name enrolled in Centrify Vault. eg. (root or DOMAIN/Administrator)'),
+        },
+        {
+            'id': 'system-name',
+            'label': _('System Name'),
+            'type': 'string',
+            'help_text': _('Machine Name enrolled with in Centrify Portal'),
+        },
+    ],
+    'required': ['url', 'account-name', 'system-name', 'client_id', 'client_password'],
+}
+
+
+# generate bearer token to authenticate with PAS portal, Input : Client ID, Client Secret
+def handle_auth(**kwargs):
+    post_data = {"grant_type": "client_credentials", "scope": kwargs['oauth_scope']}
+    response = requests.post(kwargs['endpoint'], data=post_data, auth=(kwargs['client_id'], kwargs['client_password']), verify=True, timeout=(5, 30))
+    raise_for_status(response)
+    try:
+        return response.json()['access_token']
+    except KeyError:
+        raise RuntimeError('OAuth request to tenant was unsuccessful')
+
+
+# fetch the ID of system with RedRock query, Input : System Name, Account Name
+def get_ID(**kwargs):
+    endpoint = urljoin(kwargs['url'], '/Redrock/query')
+    name = " Name='{0}' and User='{1}'".format(kwargs['system_name'], kwargs['acc_name'])
+    query = 'Select ID from VaultAccount where {0}'.format(name)
+    post_headers = {"Authorization": "Bearer " + kwargs['access_token'], "X-CENTRIFY-NATIVE-CLIENT": "true"}
+    response = requests.post(endpoint, json={'Script': query}, headers=post_headers, verify=True, timeout=(5, 30))
+    raise_for_status(response)
+    try:
+        result_str = response.json()["Result"]["Results"]
+        return result_str[0]["Row"]["ID"]
+    except (IndexError, KeyError):
+        raise RuntimeError("Error Detected!! Check the Inputs")
+
+
+# CheckOut Password from Centrify Vault, Input : ID
+def get_passwd(**kwargs):
+    endpoint = urljoin(kwargs['url'], '/ServerManage/CheckoutPassword')
+    post_headers = {"Authorization": "Bearer " + kwargs['access_token'], "X-CENTRIFY-NATIVE-CLIENT": "true"}
+    response = requests.post(endpoint, json={'ID': kwargs['acc_id']}, headers=post_headers, verify=True, timeout=(5, 30))
+    raise_for_status(response)
+    try:
+        return response.json()["Result"]["Password"]
+    except KeyError:
+        raise RuntimeError("Password Not Found")
+
+
+def centrify_backend(**kwargs):
+    url = kwargs.get('url')
+    acc_name = kwargs.get('account-name')
+    system_name = kwargs.get('system-name')
+    client_id = kwargs.get('client_id')
+    client_password = kwargs.get('client_password')
+    app_id = kwargs.get('oauth_application_id', 'awx')
+    endpoint = urljoin(url, f'/oauth2/token/{app_id}')
+    endpoint = {'endpoint': endpoint, 'client_id': client_id, 'client_password': client_password, 'oauth_scope': kwargs.get('oauth_scope', 'awx')}
+    token = handle_auth(**endpoint)
+    get_id_args = {'system_name': system_name, 'acc_name': acc_name, 'url': url, 'access_token': token}
+    acc_id = get_ID(**get_id_args)
+    get_pwd_args = {'url': url, 'acc_id': acc_id, 'access_token': token}
+    return get_passwd(**get_pwd_args)
+
+
+centrify_plugin = CredentialPlugin('Centrify Vault Credential Provider Lookup', inputs=pas_inputs, backend=centrify_backend)

awx/main/credential_plugins/conjur.py

@@ -3,7 +3,7 @@ from .plugin import CredentialPlugin, CertFiles, raise_for_status
 import base64
 from urllib.parse import urljoin, quote
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 import requests
 
 

awx/main/credential_plugins/dsv.py

@@ -1,7 +1,7 @@
 from .plugin import CredentialPlugin
 
 from django.conf import settings
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from thycotic.secrets.vault import SecretsVault
 
 

awx/main/credential_plugins/hashivault.py

@@ -6,7 +6,7 @@ from urllib.parse import urljoin
 from .plugin import CredentialPlugin, CertFiles, raise_for_status
 
 import requests
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 base_inputs = {
     'fields': [

awx/main/credential_plugins/tss.py

@@ -1,5 +1,5 @@
 from .plugin import CredentialPlugin
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from thycotic.secrets.server import PasswordGrantAuthorizer, SecretServer, ServerSecret
 

awx/main/dispatch/control.py

@@ -42,7 +42,7 @@ class Control(object):
         return f"reply_to_{str(uuid.uuid4()).replace('-','_')}"
 
     def control_with_reply(self, command, timeout=5):
-        logger.warn('checking {} {} for {}'.format(self.service, command, self.queuename))
+        logger.warning('checking {} {} for {}'.format(self.service, command, self.queuename))
         reply_queue = Control.generate_reply_queue_name()
         self.result = None
 

awx/main/dispatch/periodic.py

@@ -6,7 +6,8 @@ from multiprocessing import Process
 from django.conf import settings
 from django.db import connections
 from schedule import Scheduler
-from django_guid.middleware import GuidMiddleware
+from django_guid import set_guid
+from django_guid.utils import generate_guid
 
 from awx.main.dispatch.worker import TaskWorker
 
@@ -19,20 +20,20 @@ class Scheduler(Scheduler):
 
         def run():
             ppid = os.getppid()
-            logger.warn('periodic beat started')
+            logger.warning('periodic beat started')
             while True:
                 if os.getppid() != ppid:
                     # if the parent PID changes, this process has been orphaned
                     # via e.g., segfault or sigkill, we should exit too
                     pid = os.getpid()
-                    logger.warn(f'periodic beat exiting gracefully pid:{pid}')
+                    logger.warning(f'periodic beat exiting gracefully pid:{pid}')
                     raise SystemExit()
                 try:
                     for conn in connections.all():
                         # If the database connection has a hiccup, re-establish a new
                         # connection
                         conn.close_if_unusable_or_obsolete()
-                    GuidMiddleware.set_guid(GuidMiddleware._generate_guid())
+                    set_guid(generate_guid())
                     self.run_pending()
                 except Exception:
                     logger.exception('encountered an error while scheduling periodic tasks')

awx/main/dispatch/pool.py

@@ -16,13 +16,13 @@ from queue import Full as QueueFull, Empty as QueueEmpty
 from django.conf import settings
 from django.db import connection as django_connection, connections
 from django.core.cache import cache as django_cache
-from django_guid.middleware import GuidMiddleware
+from django_guid import set_guid
 from jinja2 import Template
 import psutil
 
 from awx.main.models import UnifiedJob
 from awx.main.dispatch import reaper
-from awx.main.utils.common import convert_mem_str_to_bytes
+from awx.main.utils.common import convert_mem_str_to_bytes, get_mem_effective_capacity
 
 if 'run_callback_receiver' in sys.argv:
     logger = logging.getLogger('awx.main.commands.run_callback_receiver')
@@ -142,7 +142,7 @@ class PoolWorker(object):
                 # when this occurs, it's _fine_ to ignore this KeyError because
                 # the purpose of self.managed_tasks is to just track internal
                 # state of which events are *currently* being processed.
-                logger.warn('Event UUID {} appears to be have been duplicated.'.format(uuid))
+                logger.warning('Event UUID {} appears to be have been duplicated.'.format(uuid))
 
     @property
     def current_task(self):
@@ -291,8 +291,8 @@ class WorkerPool(object):
                 pass
             except Exception:
                 tb = traceback.format_exc()
-                logger.warn("could not write to queue %s" % preferred_queue)
-                logger.warn("detail: {}".format(tb))
+                logger.warning("could not write to queue %s" % preferred_queue)
+                logger.warning("detail: {}".format(tb))
             write_attempt_order.append(preferred_queue)
         logger.error("could not write payload to any queue, attempted order: {}".format(write_attempt_order))
         return None
@@ -324,8 +324,9 @@ class AutoscalePool(WorkerPool):
                 total_memory_gb = convert_mem_str_to_bytes(settings_absmem) // 2**30
             else:
                 total_memory_gb = (psutil.virtual_memory().total >> 30) + 1  # noqa: round up
-            # 5 workers per GB of total memory
-            self.max_workers = total_memory_gb * 5
+
+            # Get same number as max forks based on memory, this function takes memory as bytes
+            self.max_workers = get_mem_effective_capacity(total_memory_gb * 2**30)
 
         # max workers can't be less than min_workers
         self.max_workers = max(self.min_workers, self.max_workers)
@@ -435,7 +436,7 @@ class AutoscalePool(WorkerPool):
 
     def write(self, preferred_queue, body):
         if 'guid' in body:
-            GuidMiddleware.set_guid(body['guid'])
+            set_guid(body['guid'])
         try:
             # when the cluster heartbeat occurs, clean up internally
             if isinstance(body, dict) and 'cluster_node_heartbeat' in body['task']:

awx/main/dispatch/publish.py

@@ -5,7 +5,7 @@ import json
 from uuid import uuid4
 
 from django.conf import settings
-from django_guid.middleware import GuidMiddleware
+from django_guid import get_guid
 
 from . import pg_bus_conn
 
@@ -76,7 +76,7 @@ class task:
                     logger.error(msg)
                     raise ValueError(msg)
                 obj = {'uuid': task_id, 'args': args, 'kwargs': kwargs, 'task': cls.name}
-                guid = GuidMiddleware.get_guid()
+                guid = get_guid()
                 if guid:
                     obj['guid'] = guid
                 obj.update(**kw)

awx/main/dispatch/worker/base.py

@@ -60,7 +60,7 @@ class AWXConsumerBase(object):
         return f'listening on {self.queues}'
 
     def control(self, body):
-        logger.warn(f'Received control signal:\n{body}')
+        logger.warning(f'Received control signal:\n{body}')
         control = body.get('control')
         if control in ('status', 'running'):
             reply_queue = body['reply_to']
@@ -118,7 +118,7 @@ class AWXConsumerBase(object):
 
     def stop(self, signum, frame):
         self.should_stop = True
-        logger.warn('received {}, stopping'.format(signame(signum)))
+        logger.warning('received {}, stopping'.format(signame(signum)))
         self.worker.on_stop()
         raise SystemExit()
 
@@ -153,7 +153,7 @@ class AWXConsumerPG(AWXConsumerBase):
                     if self.should_stop:
                         return
             except psycopg2.InterfaceError:
-                logger.warn("Stale Postgres message bus connection, reconnecting")
+                logger.warning("Stale Postgres message bus connection, reconnecting")
                 continue
 
 

awx/main/dispatch/worker/callback.py

@@ -9,7 +9,7 @@ from django.conf import settings
 from django.utils.timezone import now as tz_now
 from django.db import DatabaseError, OperationalError, connection as django_connection
 from django.db.utils import InterfaceError, InternalError
-from django_guid.middleware import GuidMiddleware
+from django_guid import set_guid
 
 import psutil
 
@@ -184,7 +184,7 @@ class CallbackBrokerWorker(BaseWorker):
                 if body.get('event') == 'EOF':
                     try:
                         if 'guid' in body:
-                            GuidMiddleware.set_guid(body['guid'])
+                            set_guid(body['guid'])
                         final_counter = body.get('final_counter', 0)
                         logger.info('Event processing is finished for Job {}, sending notifications'.format(job_identifier))
                         # EOF events are sent when stdout for the running task is
@@ -208,7 +208,7 @@ class CallbackBrokerWorker(BaseWorker):
                         logger.exception('Worker failed to emit notifications: Job {}'.format(job_identifier))
                     finally:
                         self.subsystem_metrics.inc('callback_receiver_events_in_memory', -1)
-                        GuidMiddleware.set_guid('')
+                        set_guid('')
                     return
 
                 skip_websocket_message = body.pop('skip_websocket_message', False)

awx/main/dispatch/worker/task.py

@@ -7,7 +7,7 @@ import traceback
 from kubernetes.config import kube_config
 
 from django.conf import settings
-from django_guid.middleware import GuidMiddleware
+from django_guid import set_guid
 
 from awx.main.tasks.system import dispatch_startup, inform_cluster_of_shutdown
 
@@ -54,7 +54,7 @@ class TaskWorker(BaseWorker):
         args = body.get('args', [])
         kwargs = body.get('kwargs', {})
         if 'guid' in body:
-            GuidMiddleware.set_guid(body.pop('guid'))
+            set_guid(body.pop('guid'))
         _call = TaskWorker.resolve_callable(task)
         if inspect.isclass(_call):
             # the callable is a class, e.g., RunJob; instantiate and

awx/main/fields.py

@@ -11,7 +11,6 @@ from jinja2 import sandbox, StrictUndefined
 from jinja2.exceptions import UndefinedError, TemplateSyntaxError, SecurityError
 
 # Django
-from django.contrib.postgres.fields import JSONField as upstream_JSONBField
 from django.core import exceptions as django_exceptions
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db.models.signals import (
@@ -28,17 +27,15 @@ from django.db.models.fields.related_descriptors import (
     ReverseManyToOneDescriptor,
     create_forward_many_to_many_manager,
 )
-from django.utils.encoding import smart_text
+from django.utils.encoding import smart_str
+from django.db.models import JSONField
 from django.utils.functional import cached_property
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 # jsonschema
 from jsonschema import Draft4Validator, FormatChecker
 import jsonschema.exceptions
 
-# Django-JSONField
-from jsonfield import JSONField as upstream_JSONField
-
 # DRF
 from rest_framework import serializers
 
@@ -52,9 +49,9 @@ from awx.main import utils
 
 
 __all__ = [
+    'JSONBlob',
     'AutoOneToOneField',
     'ImplicitRoleField',
-    'JSONField',
     'SmartFilterField',
     'OrderedManyToManyField',
     'update_role_parentage_for_instance',
@@ -71,34 +68,9 @@ def __enum_validate__(validator, enums, instance, schema):
 Draft4Validator.VALIDATORS['enum'] = __enum_validate__
 
 
-class JSONField(upstream_JSONField):
-    def db_type(self, connection):
-        return 'text'
-
-    def from_db_value(self, value, expression, connection):
-        if value in {'', None} and not self.null:
-            return {}
-        return super(JSONField, self).from_db_value(value, expression, connection)
-
-
-class JSONBField(upstream_JSONBField):
-    def get_prep_lookup(self, lookup_type, value):
-        if isinstance(value, str) and value == "null":
-            return 'null'
-        return super(JSONBField, self).get_prep_lookup(lookup_type, value)
-
-    def get_db_prep_value(self, value, connection, prepared=False):
-        if connection.vendor == 'sqlite':
-            # sqlite (which we use for tests) does not support jsonb;
-            return json.dumps(value, cls=DjangoJSONEncoder)
-        return super(JSONBField, self).get_db_prep_value(value, connection, prepared)
-
-    def from_db_value(self, value, expression, connection):
-        # Work around a bug in django-jsonfield
-        # https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos
-        if isinstance(value, str):
-            return json.loads(value)
-        return value
+class JSONBlob(JSONField):
+    def get_internal_type(self):
+        return "TextField"
 
 
 # Based on AutoOneToOneField from django-annoying:
@@ -140,7 +112,7 @@ def resolve_role_field(obj, field):
         # use extremely generous duck typing to accomidate all possible forms
         # of the model that may be used during various migrations
         if obj._meta.model_name != 'role' or obj._meta.app_label != 'main':
-            raise Exception(smart_text('{} refers to a {}, not a Role'.format(field, type(obj))))
+            raise Exception(smart_str('{} refers to a {}, not a Role'.format(field, type(obj))))
         ret.append(obj.id)
     else:
         if type(obj) is ManyToManyDescriptor:
@@ -385,7 +357,7 @@ class SmartFilterField(models.TextField):
         return super(SmartFilterField, self).get_prep_value(value)
 
 
-class JSONSchemaField(JSONBField):
+class JSONSchemaField(models.JSONField):
     """
     A JSONB field that self-validates against a defined JSON schema
     (http://json-schema.org).  This base class is intended to be overwritten by
@@ -398,8 +370,13 @@ class JSONSchemaField(JSONBField):
     # validation
     empty_values = (None, '')
 
+    def __init__(self, encoder=None, decoder=None, **options):
+        if encoder is None:
+            encoder = DjangoJSONEncoder
+        super().__init__(encoder=encoder, decoder=decoder, **options)
+
     def get_default(self):
-        return copy.deepcopy(super(JSONBField, self).get_default())
+        return copy.deepcopy(super(models.JSONField, self).get_default())
 
     def schema(self, model_instance):
         raise NotImplementedError()

awx/main/management/commands/cleanup_jobs.py

@@ -11,13 +11,12 @@ import re
 # Django
 from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction, connection
+from django.db.models import Min, Max
+from django.db.models.signals import pre_save, post_save, pre_delete, post_delete, m2m_changed
 from django.utils.timezone import now
 
 # AWX
 from awx.main.models import Job, AdHocCommand, ProjectUpdate, InventoryUpdate, SystemJob, WorkflowJob, Notification
-from awx.main.signals import disable_activity_stream, disable_computed_fields
-
-from awx.main.utils.deletion import AWXCollector, pre_delete
 
 
 def unified_job_class_to_event_table_name(job_class):
@@ -80,7 +79,6 @@ class DeleteMeta:
         ).count()
 
     def identify_excluded_partitions(self):
-
         part_drop = {}
 
         for pk, status, created in self.jobs_qs:
@@ -94,7 +92,7 @@ class DeleteMeta:
         # Note that parts_no_drop _may_ contain the names of partitions that don't exist
         # This can happen when the cleanup of _unpartitioned_* logic leaves behind jobs with status pending, waiting, running. The find_jobs_to_delete() will
         # pick these jobs up.
-        self.parts_no_drop = set([k for k, v in part_drop.items() if v is False])
+        self.parts_no_drop = {k for k, v in part_drop.items() if v is False}
 
     def delete_jobs(self):
         if not self.dry_run:
@@ -116,7 +114,7 @@ class DeleteMeta:
         partitions_dt = [p for p in partitions_dt if not None]
 
         # convert datetime partition back to string partition
-        partitions_maybe_drop = set([dt_to_partition_name(tbl_name, dt) for dt in partitions_dt])
+        partitions_maybe_drop = {dt_to_partition_name(tbl_name, dt) for dt in partitions_dt}
 
         # Do not drop partition if there is a job that will not be deleted pointing at it
         self.parts_to_drop = partitions_maybe_drop - self.parts_no_drop
@@ -164,6 +162,15 @@ class Command(BaseCommand):
         parser.add_argument('--notifications', dest='only_notifications', action='store_true', default=False, help='Remove notifications')
         parser.add_argument('--workflow-jobs', default=False, action='store_true', dest='only_workflow_jobs', help='Remove workflow jobs')
 
+    def init_logging(self):
+        log_levels = dict(enumerate([logging.ERROR, logging.INFO, logging.DEBUG, 0]))
+        self.logger = logging.getLogger('awx.main.commands.cleanup_jobs')
+        self.logger.setLevel(log_levels.get(self.verbosity, 0))
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter('%(message)s'))
+        self.logger.addHandler(handler)
+        self.logger.propagate = False
+
     def cleanup(self, job_class):
         delete_meta = DeleteMeta(self.logger, job_class, self.cutoff, self.dry_run)
         skipped, deleted = delete_meta.delete()
@@ -193,7 +200,7 @@ class Command(BaseCommand):
         return (delete_meta.jobs_no_delete_count, delete_meta.jobs_to_delete_count)
 
     def _cascade_delete_job_events(self, model, pk_list):
-        if len(pk_list) > 0:
+        if pk_list:
             with connection.cursor() as cursor:
                 tblname = unified_job_class_to_event_table_name(model)
 
@@ -202,37 +209,30 @@ class Command(BaseCommand):
                 cursor.execute(f"DELETE FROM _unpartitioned_{tblname} WHERE {rel_name} IN ({pk_list_csv})")
 
     def cleanup_jobs(self):
-        skipped, deleted = 0, 0
+        batch_size = 100000
 
-        batch_size = 1000000
+        # Hack to avoid doing N+1 queries as each item in the Job query set does
+        # an individual query to get the underlying UnifiedJob.
+        Job.polymorphic_super_sub_accessors_replaced = True
 
-        while True:
-            # get queryset for available jobs to remove
-            qs = Job.objects.filter(created__lt=self.cutoff).exclude(status__in=['pending', 'waiting', 'running'])
-            # get pk list for the first N (batch_size) objects
-            pk_list = qs[0:batch_size].values_list('pk', flat=True)
-            # You cannot delete queries with sql LIMIT set, so we must
-            # create a new query from this pk_list
-            qs_batch = Job.objects.filter(pk__in=pk_list)
-            just_deleted = 0
-            if not self.dry_run:
+        skipped = (Job.objects.filter(created__gte=self.cutoff) | Job.objects.filter(status__in=['pending', 'waiting', 'running'])).count()
+
+        qs = Job.objects.select_related('unifiedjob_ptr').filter(created__lt=self.cutoff).exclude(status__in=['pending', 'waiting', 'running'])
+        if self.dry_run:
+            deleted = qs.count()
+            return skipped, deleted
+
+        deleted = 0
+        info = qs.aggregate(min=Min('id'), max=Max('id'))
+        if info['min'] is not None:
+            for start in range(info['min'], info['max'] + 1, batch_size):
+                qs_batch = qs.filter(id__gte=start, id__lte=start + batch_size)
+                pk_list = qs_batch.values_list('id', flat=True)
+
+                _, results = qs_batch.delete()
+                deleted += results['main.Job']
                 self._cascade_delete_job_events(Job, pk_list)
 
-                del_query = pre_delete(qs_batch)
-                collector = AWXCollector(del_query.db)
-                collector.collect(del_query)
-                _, models_deleted = collector.delete()
-                if models_deleted:
-                    just_deleted = models_deleted['main.Job']
-                deleted += just_deleted
-            else:
-                just_deleted = 0  # break from loop, this is dry run
-                deleted = qs.count()
-
-            if just_deleted == 0:
-                break
-
-        skipped += (Job.objects.filter(created__gte=self.cutoff) | Job.objects.filter(status__in=['pending', 'waiting', 'running'])).count()
         return skipped, deleted
 
     def cleanup_ad_hoc_commands(self):
@@ -339,15 +339,6 @@ class Command(BaseCommand):
         skipped += SystemJob.objects.filter(created__gte=self.cutoff).count()
         return skipped, deleted
 
-    def init_logging(self):
-        log_levels = dict(enumerate([logging.ERROR, logging.INFO, logging.DEBUG, 0]))
-        self.logger = logging.getLogger('awx.main.commands.cleanup_jobs')
-        self.logger.setLevel(log_levels.get(self.verbosity, 0))
-        handler = logging.StreamHandler()
-        handler.setFormatter(logging.Formatter('%(message)s'))
-        self.logger.addHandler(handler)
-        self.logger.propagate = False
-
     def cleanup_workflow_jobs(self):
         skipped, deleted = 0, 0
         workflow_jobs = WorkflowJob.objects.filter(created__lt=self.cutoff)
@@ -398,6 +389,7 @@ class Command(BaseCommand):
             self.cutoff = now() - datetime.timedelta(days=self.days)
         except OverflowError:
             raise CommandError('--days specified is too large. Try something less than 99999 (about 270 years).')
+
         model_names = ('jobs', 'ad_hoc_commands', 'project_updates', 'inventory_updates', 'management_jobs', 'workflow_jobs', 'notifications')
         models_to_cleanup = set()
         for m in model_names:
@@ -405,18 +397,28 @@ class Command(BaseCommand):
                 models_to_cleanup.add(m)
         if not models_to_cleanup:
             models_to_cleanup.update(model_names)
-        with disable_activity_stream(), disable_computed_fields():
-            for m in model_names:
-                if m in models_to_cleanup:
-                    skipped, deleted = getattr(self, 'cleanup_%s' % m)()
 
-                    func = getattr(self, 'cleanup_%s_partition' % m, None)
-                    if func:
-                        skipped_partition, deleted_partition = func()
-                        skipped += skipped_partition
-                        deleted += deleted_partition
+        # Completely disconnect all signal handlers.  This is very aggressive,
+        # but it will be ok since this command is run in its own process.  The
+        # core of the logic is borrowed from Signal.disconnect().
+        for s in (pre_save, post_save, pre_delete, post_delete, m2m_changed):
+            with s.lock:
+                del s.receivers[:]
+                s.sender_receivers_cache.clear()
 
-                    if self.dry_run:
-                        self.logger.log(99, '%s: %d would be deleted, %d would be skipped.', m.replace('_', ' '), deleted, skipped)
-                    else:
-                        self.logger.log(99, '%s: %d deleted, %d skipped.', m.replace('_', ' '), deleted, skipped)
+        for m in model_names:
+            if m not in models_to_cleanup:
+                continue
+
+            skipped, deleted = getattr(self, 'cleanup_%s' % m)()
+
+            func = getattr(self, 'cleanup_%s_partition' % m, None)
+            if func:
+                skipped_partition, deleted_partition = func()
+                skipped += skipped_partition
+                deleted += deleted_partition
+
+            if self.dry_run:
+                self.logger.log(99, '%s: %d would be deleted, %d would be skipped.', m.replace('_', ' '), deleted, skipped)
+            else:
+                self.logger.log(99, '%s: %d deleted, %d skipped.', m.replace('_', ' '), deleted, skipped)

awx/main/management/commands/inventory_import.py

@@ -16,7 +16,7 @@ from collections import OrderedDict
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 from django.db import connection, transaction
-from django.utils.encoding import smart_text
+from django.utils.encoding import smart_str
 
 # DRF error class to distinguish license exceptions
 from rest_framework.exceptions import PermissionDenied
@@ -79,13 +79,13 @@ class AnsibleInventoryLoader(object):
         ee = get_default_execution_environment()
 
         if settings.IS_K8S:
-            logger.warn('This command is not able to run on kubernetes-based deployment. This action should be done using the API.')
+            logger.warning('This command is not able to run on kubernetes-based deployment. This action should be done using the API.')
             sys.exit(1)
 
         if ee.credential:
             process = subprocess.run(['podman', 'image', 'exists', ee.image], capture_output=True)
             if process.returncode != 0:
-                logger.warn(
+                logger.warning(
                     f'The default execution environment (id={ee.id}, name={ee.name}, image={ee.image}) is not available on this node. '
                     'The image needs to be available locally before using this command, due to registry authentication. '
                     'To pull this image, either run a job on this node or manually pull the image.'
@@ -109,8 +109,8 @@ class AnsibleInventoryLoader(object):
 
         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout, stderr = proc.communicate()
-        stdout = smart_text(stdout)
-        stderr = smart_text(stderr)
+        stdout = smart_str(stdout)
+        stderr = smart_str(stderr)
 
         if proc.returncode != 0:
             raise RuntimeError('%s failed (rc=%d) with stdout:\n%s\nstderr:\n%s' % ('ansible-inventory', proc.returncode, stdout, stderr))
@@ -224,7 +224,7 @@ class Command(BaseCommand):
                     from_dict = instance_id
                 if instance_id:
                     break
-        return smart_text(instance_id)
+        return smart_str(instance_id)
 
     def _get_enabled(self, from_dict, default=None):
         """

awx/main/managers.py

@@ -1,12 +1,11 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
 
-import sys
 import logging
 import os
 from django.db import models
 from django.conf import settings
-
+from django.db.models.functions import Lower
 from awx.main.utils.filters import SmartFilter
 from awx.main.utils.pglock import advisory_lock
 from awx.main.utils.common import get_capacity_type
@@ -35,7 +34,7 @@ class HostManager(models.Manager):
          - Only consider results that are unique
          - Return the count of this query
         """
-        return self.order_by().exclude(inventory_sources__source='controller').values('name').distinct().count()
+        return self.order_by().exclude(inventory_sources__source='controller').values(name_lower=Lower('name')).distinct().count()
 
     def org_active_count(self, org_id):
         """Return count of active, unique hosts used by an organization.
@@ -104,10 +103,6 @@ class InstanceManager(models.Manager):
 
     def me(self):
         """Return the currently active instance."""
-        # If we are running unit tests, return a stub record.
-        if settings.IS_TESTING(sys.argv) or hasattr(sys, '_called_from_test'):
-            return self.model(id=1, hostname=settings.CLUSTER_HOST_ID, uuid=UUID_DEFAULT)
-
         node = self.filter(hostname=settings.CLUSTER_HOST_ID)
         if node.exists():
             return node[0]
@@ -247,7 +242,7 @@ class InstanceGroupManager(models.Manager):
             if t.controller_node:
                 control_groups = instance_ig_mapping.get(t.controller_node, [])
                 if not control_groups:
-                    logger.warn(f"No instance group found for {t.controller_node}, capacity consumed may be innaccurate.")
+                    logger.warning(f"No instance group found for {t.controller_node}, capacity consumed may be innaccurate.")
 
             if t.status == 'waiting' or (not t.execution_node and not t.is_container_group_task):
                 # Subtract capacity from any peer groups that share instances

awx/main/middleware.py

@@ -14,7 +14,7 @@ from django.db import connection
 from django.shortcuts import redirect
 from django.apps import apps
 from django.utils.deprecation import MiddlewareMixin
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.urls import reverse, resolve
 
 from awx.main.utils.named_url_graph import generate_graph, GraphNode
@@ -103,7 +103,7 @@ def _customize_graph():
 
 
 class URLModificationMiddleware(MiddlewareMixin):
-    def __init__(self, get_response=None):
+    def __init__(self, get_response):
         models = [m for m in apps.get_app_config('main').get_models() if hasattr(m, 'get_absolute_url')]
         generate_graph(models)
         _customize_graph()

awx/main/migrations/0001_initial.py

@@ -7,7 +7,6 @@ from __future__ import unicode_literals
 
 from django.db import migrations, models
 import django.utils.timezone
-import jsonfield.fields
 import django.db.models.deletion
 from django.conf import settings
 import taggit.managers
@@ -70,7 +69,7 @@ class Migration(migrations.Migration):
                         ],
                     ),
                 ),
-                ('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
+                ('event_data', awx.main.fields.JSONBlob(default=dict, blank=True)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('counter', models.PositiveIntegerField(default=0)),
@@ -433,7 +432,7 @@ class Migration(migrations.Migration):
                         ],
                     ),
                 ),
-                ('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
+                ('event_data', awx.main.fields.JSONBlob(default=dict, blank=True)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('host_name', models.CharField(default='', max_length=1024, editable=False)),
@@ -623,7 +622,7 @@ class Migration(migrations.Migration):
                 ('dtend', models.DateTimeField(default=None, null=True, editable=False)),
                 ('rrule', models.CharField(max_length=255)),
                 ('next_run', models.DateTimeField(default=None, null=True, editable=False)),
-                ('extra_data', jsonfield.fields.JSONField(default=dict, blank=True)),
+                ('extra_data', models.JSONField(default=dict, null=True, blank=True)),
                 (
                     'created_by',
                     models.ForeignKey(
@@ -751,7 +750,7 @@ class Migration(migrations.Migration):
                 ('elapsed', models.DecimalField(editable=False, max_digits=12, decimal_places=3)),
                 ('job_args', models.TextField(default='', editable=False, blank=True)),
                 ('job_cwd', models.CharField(default='', max_length=1024, editable=False, blank=True)),
-                ('job_env', jsonfield.fields.JSONField(default=dict, editable=False, blank=True)),
+                ('job_env', models.JSONField(default=dict, editable=False, null=True, blank=True)),
                 ('job_explanation', models.TextField(default='', editable=False, blank=True)),
                 ('start_args', models.TextField(default='', editable=False, blank=True)),
                 ('result_stdout_text', models.TextField(default='', editable=False, blank=True)),
@@ -1035,7 +1034,7 @@ class Migration(migrations.Migration):
                 ('host_config_key', models.CharField(default='', max_length=1024, blank=True)),
                 ('ask_variables_on_launch', models.BooleanField(default=False)),
                 ('survey_enabled', models.BooleanField(default=False)),
-                ('survey_spec', jsonfield.fields.JSONField(default=dict, blank=True)),
+                ('survey_spec', models.JSONField(default=dict, blank=True)),
             ],
             options={
                 'ordering': ('name',),

awx/main/migrations/0002_squashed_v300_release.py

@@ -12,7 +12,6 @@ import django.db.models.deletion
 from django.conf import settings
 from django.utils.timezone import now
 
-import jsonfield.fields
 import taggit.managers
 
 
@@ -199,7 +198,7 @@ class Migration(migrations.Migration):
                 ),
                 ('recipients', models.TextField(default='', editable=False, blank=True)),
                 ('subject', models.TextField(default='', editable=False, blank=True)),
-                ('body', jsonfield.fields.JSONField(default=dict, blank=True)),
+                ('body', models.JSONField(default=dict, null=True, blank=True)),
             ],
             options={
                 'ordering': ('pk',),
@@ -230,7 +229,7 @@ class Migration(migrations.Migration):
                         ],
                     ),
                 ),
-                ('notification_configuration', jsonfield.fields.JSONField(default=dict)),
+                ('notification_configuration', models.JSONField(default=dict)),
                 (
                     'created_by',
                     models.ForeignKey(
@@ -324,9 +323,7 @@ class Migration(migrations.Migration):
                 ('module', models.CharField(max_length=128)),
                 (
                     'facts',
-                    awx.main.fields.JSONBField(
-                        default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True
-                    ),
+                    models.JSONField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
                 ),
                 (
                     'host',

awx/main/migrations/0004_squashed_v310_release.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 from django.db import migrations, models
 import awx.main.models.notifications
-import jsonfield.fields
 import django.db.models.deletion
 import awx.main.models.workflow
 import awx.main.fields
@@ -221,7 +220,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobnode',
             name='char_prompts',
-            field=jsonfield.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
@@ -260,7 +259,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='char_prompts',
-            field=jsonfield.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
@@ -308,12 +307,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='job',
             name='artifacts',
-            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
             name='ancestor_artifacts',
-            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         # Job timeout settings
         migrations.AddField(
@@ -381,9 +380,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='project',
             name='playbook_files',
-            field=jsonfield.fields.JSONField(
-                default=[], help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True
-            ),
+            field=models.JSONField(default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True),
         ),
         # Job events to stdout
         migrations.AddField(
@@ -539,7 +536,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjob',
             name='survey_passwords',
-            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplate',
@@ -549,85 +546,83 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='survey_spec',
-            field=jsonfield.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, blank=True),
         ),
         # JSON field changes
         migrations.AlterField(
             model_name='adhoccommandevent',
             name='event_data',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=awx.main.fields.JSONBlob(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='job',
             name='artifacts',
-            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         migrations.AlterField(
             model_name='job',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         migrations.AlterField(
             model_name='jobevent',
             name='event_data',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=awx.main.fields.JSONBlob(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='survey_spec',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='notification',
             name='body',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
         migrations.AlterField(
             model_name='notificationtemplate',
             name='notification_configuration',
-            field=awx.main.fields.JSONField(default=dict),
+            field=models.JSONField(default=dict),
         ),
         migrations.AlterField(
             model_name='project',
             name='playbook_files',
-            field=awx.main.fields.JSONField(
-                default=[], help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True
-            ),
+            field=models.JSONField(default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='schedule',
             name='extra_data',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
         migrations.AlterField(
             model_name='unifiedjob',
             name='job_env',
-            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjob',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobnode',
             name='ancestor_artifacts',
-            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobnode',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplate',
             name='survey_spec',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplatenode',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
         # Job Project Update
         migrations.AddField(

awx/main/migrations/0006_v320_release.py

@@ -108,14 +108,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='fact',
             name='facts',
-            field=awx.main.fields.JSONBField(
-                default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True
-            ),
+            field=models.JSONField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
         ),
         migrations.AddField(
             model_name='host',
             name='ansible_facts',
-            field=awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
+            field=models.JSONField(default=dict, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
         ),
         migrations.AddField(
             model_name='host',
@@ -177,8 +175,8 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='project',
             name='inventory_files',
-            field=awx.main.fields.JSONField(
-                default=[],
+            field=models.JSONField(
+                default=list,
                 help_text='Suggested list of content that could be Ansible inventory in the project',
                 verbose_name='Inventory Files',
                 editable=False,

awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-from django.db import migrations
-import awx.main.fields
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -15,6 +14,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='activitystream',
             name='setting',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
     ]

awx/main/migrations/0014_v330_saved_launchtime_configs.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
         migrations.AddField(
             model_name='schedule',
@@ -37,7 +37,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
@@ -47,12 +47,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobnode',
             name='extra_data',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
@@ -62,12 +62,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='extra_data',
-            field=awx.main.fields.JSONField(default=dict, blank=True),
+            field=models.JSONField(default=dict, null=True, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
         # Run data migration before removing the old credential field
         migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrations.RunPython.noop),
@@ -85,9 +85,9 @@ class Migration(migrations.Migration):
             name='JobLaunchConfig',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('extra_data', awx.main.fields.JSONField(blank=True, default=dict)),
-                ('survey_passwords', awx.main.fields.JSONField(blank=True, default=dict, editable=False)),
-                ('char_prompts', awx.main.fields.JSONField(blank=True, default=dict)),
+                ('extra_data', models.JSONField(blank=True, null=True, default=dict)),
+                ('survey_passwords', models.JSONField(blank=True, null=True, default=dict, editable=False)),
+                ('char_prompts', models.JSONField(blank=True, null=True, default=dict)),
                 ('credentials', models.ManyToManyField(related_name='joblaunchconfigs', to='main.Credential')),
                 (
                     'inventory',

awx/main/migrations/0018_v330_add_additional_stdout_events.py

@@ -2,10 +2,11 @@
 # Generated by Django 1.11.7 on 2017-12-14 15:13
 from __future__ import unicode_literals
 
-import awx.main.fields
 from django.db import migrations, models
 import django.db.models.deletion
 
+import awx.main.fields
+
 
 class Migration(migrations.Migration):
 
@@ -20,7 +21,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
+                ('event_data', awx.main.fields.JSONBlob(blank=True, default=dict)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
                 ('counter', models.PositiveIntegerField(default=0, editable=False)),
                 ('stdout', models.TextField(default='', editable=False)),
@@ -84,7 +85,7 @@ class Migration(migrations.Migration):
                         max_length=100,
                     ),
                 ),
-                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
+                ('event_data', awx.main.fields.JSONBlob(blank=True, default=dict)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
@@ -114,7 +115,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
+                ('event_data', awx.main.fields.JSONBlob(blank=True, default=dict)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
                 ('counter', models.PositiveIntegerField(default=0, editable=False)),
                 ('stdout', models.TextField(default='', editable=False)),

awx/main/migrations/0020_v330_instancegroup_policies.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 from django.db import migrations, models
 from decimal import Decimal
-import awx.main.fields
 
 
 class Migration(migrations.Migration):
@@ -16,8 +15,8 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='instancegroup',
             name='policy_instance_list',
-            field=awx.main.fields.JSONField(
-                default=[], help_text='List of exact-match Instances that will always be automatically assigned to this group', blank=True
+            field=models.JSONField(
+                default=list, help_text='List of exact-match Instances that will always be automatically assigned to this group', blank=True
             ),
         ),
         migrations.AddField(

awx/main/migrations/0025_v330_add_oauth_activity_stream_registrar.py

@@ -29,7 +29,7 @@ class Migration(migrations.Migration):
                 ('client_id', models.CharField(db_index=True, default=oauth2_provider.generators.generate_client_id, max_length=100, unique=True)),
                 (
                     'redirect_uris',
-                    models.TextField(blank=True, help_text='Allowed URIs list, space separated', validators=[oauth2_provider.validators.validate_uris]),
+                    models.TextField(blank=True, help_text='Allowed URIs list, space separated'),
                 ),
                 ('client_type', models.CharField(choices=[('confidential', 'Confidential'), ('public', 'Public')], max_length=32)),
                 (

awx/main/migrations/0038_v330_add_deleted_activitystream_actor.py

@@ -2,9 +2,7 @@
 # Generated by Django 1.11.11 on 2018-05-21 19:51
 from __future__ import unicode_literals
 
-import awx.main.fields
-import awx.main.models.activity_stream
-from django.db import migrations
+from django.db import models, migrations
 
 
 class Migration(migrations.Migration):
@@ -17,6 +15,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='activitystream',
             name='deleted_actor',
-            field=awx.main.fields.JSONField(null=True),
+            field=models.JSONField(null=True),
         ),
     ]

awx/main/migrations/0053_v340_workflow_inventory.py

@@ -17,7 +17,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjob',
             name='char_prompts',
-            field=awx.main.fields.JSONField(blank=True, default=dict),
+            field=models.JSONField(blank=True, null=True, default=dict),
         ),
         migrations.AddField(
             model_name='workflowjob',

awx/main/migrations/0085_v360_add_notificationtemplate_messages.py

@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 
 from django.db import migrations, models
 
-import awx.main.fields
 import awx.main.models.notifications
 
 
@@ -18,7 +17,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='notificationtemplate',
             name='messages',
-            field=awx.main.fields.JSONField(
+            field=models.JSONField(
                 default=awx.main.models.notifications.NotificationTemplate.default_messages,
                 help_text='Optional custom messages for notification template.',
                 null=True,

awx/main/migrations/0090_v360_WFJT_prompts.py

@@ -24,7 +24,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='char_prompts',
-            field=awx.main.fields.JSONField(blank=True, default=dict),
+            field=models.JSONField(blank=True, null=True, default=dict),
         ),
         migrations.AlterField(
             model_name='joblaunchconfig',

awx/main/migrations/0129_unifiedjob_installed_collections.py

@@ -1,7 +1,6 @@
 # Generated by Django 2.2.16 on 2021-02-16 20:27
 
-import awx.main.fields
-from django.db import migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -14,7 +13,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjob',
             name='installed_collections',
-            field=awx.main.fields.JSONBField(
+            field=models.JSONField(
                 blank=True, default=dict, editable=False, help_text='The Collections names and versions installed in the execution environment.'
             ),
         ),

awx/main/migrations/0150_rename_inv_sources_inv_updates.py

@@ -15,10 +15,10 @@ def forwards(apps, schema_editor):
 
     r = InventoryUpdate.objects.filter(source='tower').update(source='controller')
     if r:
-        logger.warn(f'Renamed {r} tower inventory updates to controller')
+        logger.warning(f'Renamed {r} tower inventory updates to controller')
     InventorySource.objects.filter(source='tower').update(source='controller')
     if r:
-        logger.warn(f'Renamed {r} tower inventory sources to controller')
+        logger.warning(f'Renamed {r} tower inventory sources to controller')
 
     CredentialType = apps.get_model('main', 'CredentialType')
 
@@ -32,7 +32,7 @@ def forwards(apps, schema_editor):
         registry_type = ManagedCredentialType.registry.get('controller')
         if not registry_type:
             raise RuntimeError('Excpected to find controller credential, this may need to be edited in the future!')
-        logger.warn('Renaming the Ansible Tower credential type for existing install')
+        logger.warning('Renaming the Ansible Tower credential type for existing install')
         tower_type.name = registry_type.name  # sensitive to translations
         tower_type.namespace = 'controller'  # if not done, will error setup_tower_managed_defaults
         tower_type.save(update_fields=['name', 'namespace'])
@@ -46,10 +46,10 @@ def backwards(apps, schema_editor):
 
     r = InventoryUpdate.objects.filter(source='controller').update(source='tower')
     if r:
-        logger.warn(f'Renamed {r} controller inventory updates to tower')
+        logger.warning(f'Renamed {r} controller inventory updates to tower')
     r = InventorySource.objects.filter(source='controller').update(source='tower')
     if r:
-        logger.warn(f'Renamed {r} controller inventory sources to tower')
+        logger.warning(f'Renamed {r} controller inventory sources to tower')
 
     CredentialType = apps.get_model('main', 'CredentialType')
 

awx/main/migrations/_hg_removal.py

@@ -14,4 +14,4 @@ def delete_hg_scm(apps, schema_editor):
     update_ct = Project.objects.filter(scm_type='hg').update(scm_type='')
 
     if update_ct:
-        logger.warn('Changed {} mercurial projects to manual, deprecation period ended'.format(update_ct))
+        logger.warning('Changed {} mercurial projects to manual, deprecation period ended'.format(update_ct))

awx/main/migrations/_inventory_source.py

@@ -1,6 +1,6 @@
 import logging
 
-from django.utils.encoding import smart_text
+from django.utils.encoding import smart_str
 
 from awx.main.utils.common import set_current_apps
 from awx.main.utils.common import parse_yaml_or_json
@@ -19,7 +19,7 @@ def _get_instance_id(from_dict, new_id, default=''):
             break
         instance_id = from_dict.get(key, default)
         from_dict = instance_id
-    return smart_text(instance_id)
+    return smart_str(instance_id)
 
 
 def _get_instance_id_for_upgrade(host, new_id):
@@ -35,7 +35,7 @@ def _get_instance_id_for_upgrade(host, new_id):
         return None
     if len(new_id) > 255:
         # this should never happen
-        logger.warn('Computed instance id "{}"" for host {}-{} is too long'.format(new_id_value, host.name, host.pk))
+        logger.warning('Computed instance id "{}"" for host {}-{} is too long'.format(new_id_value, host.name, host.pk))
         return None
     return new_id_value
 
@@ -47,7 +47,7 @@ def set_new_instance_id(apps, source, new_id):
     id_from_settings = getattr(settings, '{}_INSTANCE_ID_VAR'.format(source.upper()))
     if id_from_settings != new_id:
         # User applied an instance ID themselves, so nope on out of there
-        logger.warn('You have an instance ID set for {}, not migrating'.format(source))
+        logger.warning('You have an instance ID set for {}, not migrating'.format(source))
         return
     logger.debug('Migrating inventory instance_id for {} to {}'.format(source, new_id))
     Host = apps.get_model('main', 'Host')

awx/main/migrations/_inventory_source_vars.py

@@ -2,7 +2,7 @@ import json
 import re
 import logging
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.utils.encoding import iri_to_uri
 
 

awx/main/migrations/_squashed_30.py

@@ -2,7 +2,6 @@ from django.db import (
     migrations,
     models,
 )
-import jsonfield.fields
 import awx.main.fields
 
 from awx.main.migrations import _save_password_keys
@@ -30,7 +29,7 @@ SQUASHED_30 = {
         migrations.AddField(
             model_name='job',
             name='survey_passwords',
-            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
+            field=models.JSONField(default=dict, editable=False, null=True, blank=True),
         ),
     ],
     '0031_v302_migrate_survey_passwords': [

awx/main/models/__init__.py

@@ -3,6 +3,7 @@
 
 # Django
 from django.conf import settings  # noqa
+from django.db import connection
 from django.db.models.signals import pre_delete  # noqa
 
 # AWX
@@ -97,6 +98,93 @@ User.add_to_class('can_access_with_errors', check_user_access_with_errors)
 User.add_to_class('accessible_objects', user_accessible_objects)
 
 
+def convert_jsonfields_to_jsonb():
+    if connection.vendor != 'postgresql':
+        return
+
+    # fmt: off
+    fields = [  # Table name, expensive or not, tuple of column names
+        ('conf_setting', False, (
+            'value',
+        )),
+        ('main_instancegroup', False, (
+            'policy_instance_list',
+        )),
+        ('main_jobtemplate', False, (
+            'survey_spec',
+        )),
+        ('main_notificationtemplate', False, (
+            'notification_configuration',
+            'messages',
+        )),
+        ('main_project', False, (
+            'playbook_files',
+            'inventory_files',
+        )),
+        ('main_schedule', False, (
+            'extra_data',
+            'char_prompts',
+            'survey_passwords',
+        )),
+        ('main_workflowjobtemplate', False, (
+            'survey_spec',
+            'char_prompts',
+        )),
+        ('main_workflowjobtemplatenode', False, (
+            'char_prompts',
+            'extra_data',
+            'survey_passwords',
+        )),
+        ('main_activitystream', True, (
+            'setting',  # NN = NOT NULL
+            'deleted_actor',
+        )),
+        ('main_job', True, (
+            'survey_passwords',  # NN
+            'artifacts',  # NN
+        )),
+        ('main_joblaunchconfig', True, (
+            'extra_data',  # NN
+            'survey_passwords',  # NN
+            'char_prompts',  # NN
+        )),
+        ('main_notification', True, (
+            'body',  # NN
+        )),
+        ('main_unifiedjob', True, (
+            'job_env',  # NN
+        )),
+        ('main_workflowjob', True, (
+            'survey_passwords',  # NN
+            'char_prompts',  # NN
+        )),
+        ('main_workflowjobnode', True, (
+            'char_prompts',  # NN
+            'ancestor_artifacts',  # NN
+            'extra_data',  # NN
+            'survey_passwords',  # NN
+        )),
+    ]
+    # fmt: on
+
+    with connection.cursor() as cursor:
+        for table, expensive, columns in fields:
+            cursor.execute(
+                """
+                select count(1) from information_schema.columns
+                where
+                  table_name = %s and
+                  column_name in %s and
+                  data_type != 'jsonb';
+                """,
+                (table, columns),
+            )
+            if cursor.fetchone()[0]:
+                from awx.main.tasks.system import migrate_json_fields
+
+                migrate_json_fields.apply_async([table, expensive, columns])
+
+
 def cleanup_created_modified_by(sender, **kwargs):
     # work around a bug in django-polymorphic that doesn't properly
     # handle cascades for reverse foreign keys on the polymorphic base model

awx/main/models/activity_stream.py

@@ -3,14 +3,13 @@
 
 # AWX
 from awx.api.versioning import reverse
-from awx.main.fields import JSONField
 from awx.main.models.base import accepts_json
 
 # Django
 from django.db import models
 from django.conf import settings
 from django.utils.encoding import smart_str
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 __all__ = ['ActivityStream']
 
@@ -36,7 +35,7 @@ class ActivityStream(models.Model):
     operation = models.CharField(max_length=13, choices=OPERATION_CHOICES)
     timestamp = models.DateTimeField(auto_now_add=True)
     changes = accepts_json(models.TextField(blank=True))
-    deleted_actor = JSONField(null=True)
+    deleted_actor = models.JSONField(null=True)
     action_node = models.CharField(
         blank=True,
         default='',
@@ -84,7 +83,7 @@ class ActivityStream(models.Model):
     o_auth2_application = models.ManyToManyField("OAuth2Application", blank=True)
     o_auth2_access_token = models.ManyToManyField("OAuth2AccessToken", blank=True)
 
-    setting = JSONField(blank=True)
+    setting = models.JSONField(default=dict, null=True, blank=True)
 
     def __str__(self):
         operation = self.operation if 'operation' in self.__dict__ else '_delayed_'

awx/main/models/ad_hoc_commands.py

@@ -9,7 +9,7 @@ from urllib.parse import urljoin
 from django.conf import settings
 from django.db import models
 from django.utils.text import Truncator
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import ValidationError
 
 # AWX

awx/main/models/base.py

@@ -4,7 +4,7 @@
 # Django
 from django.db import models
 from django.core.exceptions import ValidationError, ObjectDoesNotExist
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.utils.timezone import now
 
 # Django-Taggit

awx/main/models/credential/__init__.py

@@ -15,9 +15,9 @@ from jinja2 import sandbox
 
 # Django
 from django.db import models
-from django.utils.translation import ugettext_lazy as _, ugettext_noop
+from django.utils.translation import gettext_lazy as _, gettext_noop
 from django.core.exceptions import ValidationError
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 from django.utils.functional import cached_property
 from django.utils.timezone import now
 
@@ -230,7 +230,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     def display_inputs(self):
         field_val = self.inputs.copy()
         for k, v in field_val.items():
-            if force_text(v).startswith('$encrypted$'):
+            if force_str(v).startswith('$encrypted$'):
                 field_val[k] = '$encrypted$'
         return field_val
 
@@ -579,34 +579,34 @@ class ManagedCredentialType(SimpleNamespace):
 ManagedCredentialType(
     namespace='ssh',
     kind='ssh',
-    name=ugettext_noop('Machine'),
+    name=gettext_noop('Machine'),
     inputs={
         'fields': [
-            {'id': 'username', 'label': ugettext_noop('Username'), 'type': 'string'},
-            {'id': 'password', 'label': ugettext_noop('Password'), 'type': 'string', 'secret': True, 'ask_at_runtime': True},
-            {'id': 'ssh_key_data', 'label': ugettext_noop('SSH Private Key'), 'type': 'string', 'format': 'ssh_private_key', 'secret': True, 'multiline': True},
+            {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
+            {'id': 'password', 'label': gettext_noop('Password'), 'type': 'string', 'secret': True, 'ask_at_runtime': True},
+            {'id': 'ssh_key_data', 'label': gettext_noop('SSH Private Key'), 'type': 'string', 'format': 'ssh_private_key', 'secret': True, 'multiline': True},
             {
                 'id': 'ssh_public_key_data',
-                'label': ugettext_noop('Signed SSH Certificate'),
+                'label': gettext_noop('Signed SSH Certificate'),
                 'type': 'string',
                 'multiline': True,
                 'secret': True,
             },
-            {'id': 'ssh_key_unlock', 'label': ugettext_noop('Private Key Passphrase'), 'type': 'string', 'secret': True, 'ask_at_runtime': True},
+            {'id': 'ssh_key_unlock', 'label': gettext_noop('Private Key Passphrase'), 'type': 'string', 'secret': True, 'ask_at_runtime': True},
             {
                 'id': 'become_method',
-                'label': ugettext_noop('Privilege Escalation Method'),
+                'label': gettext_noop('Privilege Escalation Method'),
                 'type': 'string',
-                'help_text': ugettext_noop(
+                'help_text': gettext_noop(
                     'Specify a method for "become" operations. This is ' 'equivalent to specifying the --become-method ' 'Ansible parameter.'
                 ),
             },
             {
                 'id': 'become_username',
-                'label': ugettext_noop('Privilege Escalation Username'),
+                'label': gettext_noop('Privilege Escalation Username'),
                 'type': 'string',
             },
-            {'id': 'become_password', 'label': ugettext_noop('Privilege Escalation Password'), 'type': 'string', 'secret': True, 'ask_at_runtime': True},
+            {'id': 'become_password', 'label': gettext_noop('Privilege Escalation Password'), 'type': 'string', 'secret': True, 'ask_at_runtime': True},
         ],
     },
 )
@@ -614,14 +614,14 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='scm',
     kind='scm',
-    name=ugettext_noop('Source Control'),
+    name=gettext_noop('Source Control'),
     managed=True,
     inputs={
         'fields': [
-            {'id': 'username', 'label': ugettext_noop('Username'), 'type': 'string'},
-            {'id': 'password', 'label': ugettext_noop('Password'), 'type': 'string', 'secret': True},
-            {'id': 'ssh_key_data', 'label': ugettext_noop('SCM Private Key'), 'type': 'string', 'format': 'ssh_private_key', 'secret': True, 'multiline': True},
-            {'id': 'ssh_key_unlock', 'label': ugettext_noop('Private Key Passphrase'), 'type': 'string', 'secret': True},
+            {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
+            {'id': 'password', 'label': gettext_noop('Password'), 'type': 'string', 'secret': True},
+            {'id': 'ssh_key_data', 'label': gettext_noop('SCM Private Key'), 'type': 'string', 'format': 'ssh_private_key', 'secret': True, 'multiline': True},
+            {'id': 'ssh_key_unlock', 'label': gettext_noop('Private Key Passphrase'), 'type': 'string', 'secret': True},
         ],
     },
 )
@@ -629,17 +629,17 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='vault',
     kind='vault',
-    name=ugettext_noop('Vault'),
+    name=gettext_noop('Vault'),
     managed=True,
     inputs={
         'fields': [
-            {'id': 'vault_password', 'label': ugettext_noop('Vault Password'), 'type': 'string', 'secret': True, 'ask_at_runtime': True},
+            {'id': 'vault_password', 'label': gettext_noop('Vault Password'), 'type': 'string', 'secret': True, 'ask_at_runtime': True},
             {
                 'id': 'vault_id',
-                'label': ugettext_noop('Vault Identifier'),
+                'label': gettext_noop('Vault Identifier'),
                 'type': 'string',
                 'format': 'vault_id',
-                'help_text': ugettext_noop(
+                'help_text': gettext_noop(
                     'Specify an (optional) Vault ID. This is '
                     'equivalent to specifying the --vault-id '
                     'Ansible parameter for providing multiple Vault '
@@ -655,32 +655,32 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='net',
     kind='net',
-    name=ugettext_noop('Network'),
+    name=gettext_noop('Network'),
     managed=True,
     inputs={
         'fields': [
-            {'id': 'username', 'label': ugettext_noop('Username'), 'type': 'string'},
+            {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
             {
                 'id': 'password',
-                'label': ugettext_noop('Password'),
+                'label': gettext_noop('Password'),
                 'type': 'string',
                 'secret': True,
             },
-            {'id': 'ssh_key_data', 'label': ugettext_noop('SSH Private Key'), 'type': 'string', 'format': 'ssh_private_key', 'secret': True, 'multiline': True},
+            {'id': 'ssh_key_data', 'label': gettext_noop('SSH Private Key'), 'type': 'string', 'format': 'ssh_private_key', 'secret': True, 'multiline': True},
             {
                 'id': 'ssh_key_unlock',
-                'label': ugettext_noop('Private Key Passphrase'),
+                'label': gettext_noop('Private Key Passphrase'),
                 'type': 'string',
                 'secret': True,
             },
             {
                 'id': 'authorize',
-                'label': ugettext_noop('Authorize'),
+                'label': gettext_noop('Authorize'),
                 'type': 'boolean',
             },
             {
                 'id': 'authorize_password',
-                'label': ugettext_noop('Authorize Password'),
+                'label': gettext_noop('Authorize Password'),
                 'type': 'string',
                 'secret': True,
             },
@@ -695,23 +695,23 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='aws',
     kind='cloud',
-    name=ugettext_noop('Amazon Web Services'),
+    name=gettext_noop('Amazon Web Services'),
     managed=True,
     inputs={
         'fields': [
-            {'id': 'username', 'label': ugettext_noop('Access Key'), 'type': 'string'},
+            {'id': 'username', 'label': gettext_noop('Access Key'), 'type': 'string'},
             {
                 'id': 'password',
-                'label': ugettext_noop('Secret Key'),
+                'label': gettext_noop('Secret Key'),
                 'type': 'string',
                 'secret': True,
             },
             {
                 'id': 'security_token',
-                'label': ugettext_noop('STS Token'),
+                'label': gettext_noop('STS Token'),
                 'type': 'string',
                 'secret': True,
-                'help_text': ugettext_noop(
+                'help_text': gettext_noop(
                     'Security Token Service (STS) is a web service '
                     'that enables you to request temporary, '
                     'limited-privilege credentials for AWS Identity '
@@ -726,38 +726,38 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='openstack',
     kind='cloud',
-    name=ugettext_noop('OpenStack'),
+    name=gettext_noop('OpenStack'),
     managed=True,
     inputs={
         'fields': [
-            {'id': 'username', 'label': ugettext_noop('Username'), 'type': 'string'},
+            {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
             {
                 'id': 'password',
-                'label': ugettext_noop('Password (API Key)'),
+                'label': gettext_noop('Password (API Key)'),
                 'type': 'string',
                 'secret': True,
             },
             {
                 'id': 'host',
-                'label': ugettext_noop('Host (Authentication URL)'),
+                'label': gettext_noop('Host (Authentication URL)'),
                 'type': 'string',
-                'help_text': ugettext_noop('The host to authenticate with.  For example, ' 'https://openstack.business.com/v2.0/'),
+                'help_text': gettext_noop('The host to authenticate with.  For example, ' 'https://openstack.business.com/v2.0/'),
             },
             {
                 'id': 'project',
-                'label': ugettext_noop('Project (Tenant Name)'),
+                'label': gettext_noop('Project (Tenant Name)'),
                 'type': 'string',
             },
             {
                 'id': 'project_domain_name',
-                'label': ugettext_noop('Project (Domain Name)'),
+                'label': gettext_noop('Project (Domain Name)'),
                 'type': 'string',
             },
             {
                 'id': 'domain',
-                'label': ugettext_noop('Domain Name'),
+                'label': gettext_noop('Domain Name'),
                 'type': 'string',
-                'help_text': ugettext_noop(
+                'help_text': gettext_noop(
                     'OpenStack domains define administrative boundaries. '
                     'It is only needed for Keystone v3 authentication '
                     'URLs. Refer to the documentation for '
@@ -766,13 +766,13 @@ ManagedCredentialType(
             },
             {
                 'id': 'region',
-                'label': ugettext_noop('Region Name'),
+                'label': gettext_noop('Region Name'),
                 'type': 'string',
-                'help_text': ugettext_noop('For some cloud providers, like OVH, region must be specified'),
+                'help_text': gettext_noop('For some cloud providers, like OVH, region must be specified'),
             },
             {
                 'id': 'verify_ssl',
-                'label': ugettext_noop('Verify SSL'),
+                'label': gettext_noop('Verify SSL'),
                 'type': 'boolean',
                 'default': True,
             },
@@ -784,20 +784,20 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='vmware',
     kind='cloud',
-    name=ugettext_noop('VMware vCenter'),
+    name=gettext_noop('VMware vCenter'),
     managed=True,
     inputs={
         'fields': [
             {
                 'id': 'host',
-                'label': ugettext_noop('VCenter Host'),
+                'label': gettext_noop('VCenter Host'),
                 'type': 'string',
-                'help_text': ugettext_noop('Enter the hostname or IP address that corresponds ' 'to your VMware vCenter.'),
+                'help_text': gettext_noop('Enter the hostname or IP address that corresponds ' 'to your VMware vCenter.'),
             },
-            {'id': 'username', 'label': ugettext_noop('Username'), 'type': 'string'},
+            {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
             {
                 'id': 'password',
-                'label': ugettext_noop('Password'),
+                'label': gettext_noop('Password'),
                 'type': 'string',
                 'secret': True,
             },
@@ -809,20 +809,20 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='satellite6',
     kind='cloud',
-    name=ugettext_noop('Red Hat Satellite 6'),
+    name=gettext_noop('Red Hat Satellite 6'),
     managed=True,
     inputs={
         'fields': [
             {
                 'id': 'host',
-                'label': ugettext_noop('Satellite 6 URL'),
+                'label': gettext_noop('Satellite 6 URL'),
                 'type': 'string',
-                'help_text': ugettext_noop('Enter the URL that corresponds to your Red Hat ' 'Satellite 6 server. For example, https://satellite.example.org'),
+                'help_text': gettext_noop('Enter the URL that corresponds to your Red Hat ' 'Satellite 6 server. For example, https://satellite.example.org'),
             },
-            {'id': 'username', 'label': ugettext_noop('Username'), 'type': 'string'},
+            {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
             {
                 'id': 'password',
-                'label': ugettext_noop('Password'),
+                'label': gettext_noop('Password'),
                 'type': 'string',
                 'secret': True,
             },
@@ -834,21 +834,21 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='gce',
     kind='cloud',
-    name=ugettext_noop('Google Compute Engine'),
+    name=gettext_noop('Google Compute Engine'),
     managed=True,
     inputs={
         'fields': [
             {
                 'id': 'username',
-                'label': ugettext_noop('Service Account Email Address'),
+                'label': gettext_noop('Service Account Email Address'),
                 'type': 'string',
-                'help_text': ugettext_noop('The email address assigned to the Google Compute ' 'Engine service account.'),
+                'help_text': gettext_noop('The email address assigned to the Google Compute ' 'Engine service account.'),
             },
             {
                 'id': 'project',
                 'label': 'Project',
                 'type': 'string',
-                'help_text': ugettext_noop(
+                'help_text': gettext_noop(
                     'The Project ID is the GCE assigned identification. '
                     'It is often constructed as three words or two words '
                     'followed by a three-digit number. Examples: project-id-000 '
@@ -857,12 +857,12 @@ ManagedCredentialType(
             },
             {
                 'id': 'ssh_key_data',
-                'label': ugettext_noop('RSA Private Key'),
+                'label': gettext_noop('RSA Private Key'),
                 'type': 'string',
                 'format': 'ssh_private_key',
                 'secret': True,
                 'multiline': True,
-                'help_text': ugettext_noop('Paste the contents of the PEM file associated ' 'with the service account email.'),
+                'help_text': gettext_noop('Paste the contents of the PEM file associated ' 'with the service account email.'),
             },
         ],
         'required': ['username', 'ssh_key_data'],
@@ -872,36 +872,36 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='azure_rm',
     kind='cloud',
-    name=ugettext_noop('Microsoft Azure Resource Manager'),
+    name=gettext_noop('Microsoft Azure Resource Manager'),
     managed=True,
     inputs={
         'fields': [
             {
                 'id': 'subscription',
-                'label': ugettext_noop('Subscription ID'),
+                'label': gettext_noop('Subscription ID'),
                 'type': 'string',
-                'help_text': ugettext_noop('Subscription ID is an Azure construct, which is ' 'mapped to a username.'),
+                'help_text': gettext_noop('Subscription ID is an Azure construct, which is ' 'mapped to a username.'),
             },
-            {'id': 'username', 'label': ugettext_noop('Username'), 'type': 'string'},
+            {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
             {
                 'id': 'password',
-                'label': ugettext_noop('Password'),
+                'label': gettext_noop('Password'),
                 'type': 'string',
                 'secret': True,
             },
-            {'id': 'client', 'label': ugettext_noop('Client ID'), 'type': 'string'},
+            {'id': 'client', 'label': gettext_noop('Client ID'), 'type': 'string'},
             {
                 'id': 'secret',
-                'label': ugettext_noop('Client Secret'),
+                'label': gettext_noop('Client Secret'),
                 'type': 'string',
                 'secret': True,
             },
-            {'id': 'tenant', 'label': ugettext_noop('Tenant ID'), 'type': 'string'},
+            {'id': 'tenant', 'label': gettext_noop('Tenant ID'), 'type': 'string'},
             {
                 'id': 'cloud_environment',
-                'label': ugettext_noop('Azure Cloud Environment'),
+                'label': gettext_noop('Azure Cloud Environment'),
                 'type': 'string',
-                'help_text': ugettext_noop('Environment variable AZURE_CLOUD_ENVIRONMENT when' ' using Azure GovCloud or Azure stack.'),
+                'help_text': gettext_noop('Environment variable AZURE_CLOUD_ENVIRONMENT when' ' using Azure GovCloud or Azure stack.'),
             },
         ],
         'required': ['subscription'],
@@ -911,16 +911,16 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='github_token',
     kind='token',
-    name=ugettext_noop('GitHub Personal Access Token'),
+    name=gettext_noop('GitHub Personal Access Token'),
     managed=True,
     inputs={
         'fields': [
             {
                 'id': 'token',
-                'label': ugettext_noop('Token'),
+                'label': gettext_noop('Token'),
                 'type': 'string',
                 'secret': True,
-                'help_text': ugettext_noop('This token needs to come from your profile settings in GitHub'),
+                'help_text': gettext_noop('This token needs to come from your profile settings in GitHub'),
             }
         ],
         'required': ['token'],
@@ -930,16 +930,16 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='gitlab_token',
     kind='token',
-    name=ugettext_noop('GitLab Personal Access Token'),
+    name=gettext_noop('GitLab Personal Access Token'),
     managed=True,
     inputs={
         'fields': [
             {
                 'id': 'token',
-                'label': ugettext_noop('Token'),
+                'label': gettext_noop('Token'),
                 'type': 'string',
                 'secret': True,
-                'help_text': ugettext_noop('This token needs to come from your profile settings in GitLab'),
+                'help_text': gettext_noop('This token needs to come from your profile settings in GitLab'),
             }
         ],
         'required': ['token'],
@@ -949,12 +949,12 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='insights',
     kind='insights',
-    name=ugettext_noop('Insights'),
+    name=gettext_noop('Insights'),
     managed=True,
     inputs={
         'fields': [
-            {'id': 'username', 'label': ugettext_noop('Username'), 'type': 'string'},
-            {'id': 'password', 'label': ugettext_noop('Password'), 'type': 'string', 'secret': True},
+            {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
+            {'id': 'password', 'label': gettext_noop('Password'), 'type': 'string', 'secret': True},
         ],
         'required': ['username', 'password'],
     },
@@ -973,23 +973,23 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='rhv',
     kind='cloud',
-    name=ugettext_noop('Red Hat Virtualization'),
+    name=gettext_noop('Red Hat Virtualization'),
     managed=True,
     inputs={
         'fields': [
-            {'id': 'host', 'label': ugettext_noop('Host (Authentication URL)'), 'type': 'string', 'help_text': ugettext_noop('The host to authenticate with.')},
-            {'id': 'username', 'label': ugettext_noop('Username'), 'type': 'string'},
+            {'id': 'host', 'label': gettext_noop('Host (Authentication URL)'), 'type': 'string', 'help_text': gettext_noop('The host to authenticate with.')},
+            {'id': 'username', 'label': gettext_noop('Username'), 'type': 'string'},
             {
                 'id': 'password',
-                'label': ugettext_noop('Password'),
+                'label': gettext_noop('Password'),
                 'type': 'string',
                 'secret': True,
             },
             {
                 'id': 'ca_file',
-                'label': ugettext_noop('CA File'),
+                'label': gettext_noop('CA File'),
                 'type': 'string',
-                'help_text': ugettext_noop('Absolute file path to the CA file to use (optional)'),
+                'help_text': gettext_noop('Absolute file path to the CA file to use (optional)'),
             },
         ],
         'required': ['host', 'username', 'password'],
@@ -1017,38 +1017,38 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='controller',
     kind='cloud',
-    name=ugettext_noop('Red Hat Ansible Automation Platform'),
+    name=gettext_noop('Red Hat Ansible Automation Platform'),
     managed=True,
     inputs={
         'fields': [
             {
                 'id': 'host',
-                'label': ugettext_noop('Red Hat Ansible Automation Platform'),
+                'label': gettext_noop('Red Hat Ansible Automation Platform'),
                 'type': 'string',
-                'help_text': ugettext_noop('Red Hat Ansible Automation Platform base URL to authenticate with.'),
+                'help_text': gettext_noop('Red Hat Ansible Automation Platform base URL to authenticate with.'),
             },
             {
                 'id': 'username',
-                'label': ugettext_noop('Username'),
+                'label': gettext_noop('Username'),
                 'type': 'string',
-                'help_text': ugettext_noop(
+                'help_text': gettext_noop(
                     'Red Hat Ansible Automation Platform username id to authenticate as.' 'This should not be set if an OAuth token is being used.'
                 ),
             },
             {
                 'id': 'password',
-                'label': ugettext_noop('Password'),
+                'label': gettext_noop('Password'),
                 'type': 'string',
                 'secret': True,
             },
             {
                 'id': 'oauth_token',
-                'label': ugettext_noop('OAuth Token'),
+                'label': gettext_noop('OAuth Token'),
                 'type': 'string',
                 'secret': True,
-                'help_text': ugettext_noop('An OAuth token to use to authenticate with.' 'This should not be set if username/password are being used.'),
+                'help_text': gettext_noop('An OAuth token to use to authenticate with.' 'This should not be set if username/password are being used.'),
             },
-            {'id': 'verify_ssl', 'label': ugettext_noop('Verify SSL'), 'type': 'boolean', 'secret': False},
+            {'id': 'verify_ssl', 'label': gettext_noop('Verify SSL'), 'type': 'boolean', 'secret': False},
         ],
         'required': ['host'],
     },
@@ -1071,30 +1071,30 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='kubernetes_bearer_token',
     kind='kubernetes',
-    name=ugettext_noop('OpenShift or Kubernetes API Bearer Token'),
+    name=gettext_noop('OpenShift or Kubernetes API Bearer Token'),
     inputs={
         'fields': [
             {
                 'id': 'host',
-                'label': ugettext_noop('OpenShift or Kubernetes API Endpoint'),
+                'label': gettext_noop('OpenShift or Kubernetes API Endpoint'),
                 'type': 'string',
-                'help_text': ugettext_noop('The OpenShift or Kubernetes API Endpoint to authenticate with.'),
+                'help_text': gettext_noop('The OpenShift or Kubernetes API Endpoint to authenticate with.'),
             },
             {
                 'id': 'bearer_token',
-                'label': ugettext_noop('API authentication bearer token'),
+                'label': gettext_noop('API authentication bearer token'),
                 'type': 'string',
                 'secret': True,
             },
             {
                 'id': 'verify_ssl',
-                'label': ugettext_noop('Verify SSL'),
+                'label': gettext_noop('Verify SSL'),
                 'type': 'boolean',
                 'default': True,
             },
             {
                 'id': 'ssl_ca_cert',
-                'label': ugettext_noop('Certificate Authority data'),
+                'label': gettext_noop('Certificate Authority data'),
                 'type': 'string',
                 'secret': True,
                 'multiline': True,
@@ -1107,31 +1107,31 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='registry',
     kind='registry',
-    name=ugettext_noop('Container Registry'),
+    name=gettext_noop('Container Registry'),
     inputs={
         'fields': [
             {
                 'id': 'host',
-                'label': ugettext_noop('Authentication URL'),
+                'label': gettext_noop('Authentication URL'),
                 'type': 'string',
-                'help_text': ugettext_noop('Authentication endpoint for the container registry.'),
+                'help_text': gettext_noop('Authentication endpoint for the container registry.'),
                 'default': 'quay.io',
             },
             {
                 'id': 'username',
-                'label': ugettext_noop('Username'),
+                'label': gettext_noop('Username'),
                 'type': 'string',
             },
             {
                 'id': 'password',
-                'label': ugettext_noop('Password or Token'),
+                'label': gettext_noop('Password or Token'),
                 'type': 'string',
                 'secret': True,
-                'help_text': ugettext_noop('A password or token used to authenticate with'),
+                'help_text': gettext_noop('A password or token used to authenticate with'),
             },
             {
                 'id': 'verify_ssl',
-                'label': ugettext_noop('Verify SSL'),
+                'label': gettext_noop('Verify SSL'),
                 'type': 'boolean',
                 'default': True,
             },
@@ -1144,27 +1144,27 @@ ManagedCredentialType(
 ManagedCredentialType(
     namespace='galaxy_api_token',
     kind='galaxy',
-    name=ugettext_noop('Ansible Galaxy/Automation Hub API Token'),
+    name=gettext_noop('Ansible Galaxy/Automation Hub API Token'),
     inputs={
         'fields': [
             {
                 'id': 'url',
-                'label': ugettext_noop('Galaxy Server URL'),
+                'label': gettext_noop('Galaxy Server URL'),
                 'type': 'string',
-                'help_text': ugettext_noop('The URL of the Galaxy instance to connect to.'),
+                'help_text': gettext_noop('The URL of the Galaxy instance to connect to.'),
             },
             {
                 'id': 'auth_url',
-                'label': ugettext_noop('Auth Server URL'),
+                'label': gettext_noop('Auth Server URL'),
                 'type': 'string',
-                'help_text': ugettext_noop('The URL of a Keycloak server token_endpoint, if using ' 'SSO auth.'),
+                'help_text': gettext_noop('The URL of a Keycloak server token_endpoint, if using ' 'SSO auth.'),
             },
             {
                 'id': 'token',
-                'label': ugettext_noop('API Token'),
+                'label': gettext_noop('API Token'),
                 'type': 'string',
                 'secret': True,
-                'help_text': ugettext_noop('A token to use for authentication against the Galaxy instance.'),
+                'help_text': gettext_noop('A token to use for authentication against the Galaxy instance.'),
             },
         ],
         'required': ['url'],

awx/main/models/events.py

@@ -10,13 +10,13 @@ from django.db import models, DatabaseError, connection
 from django.utils.dateparse import parse_datetime
 from django.utils.text import Truncator
 from django.utils.timezone import utc, now
-from django.utils.translation import ugettext_lazy as _
-from django.utils.encoding import force_text
+from django.utils.translation import gettext_lazy as _
+from django.utils.encoding import force_str
 
 from awx.api.versioning import reverse
 from awx.main import consumers
+from awx.main.fields import JSONBlob
 from awx.main.managers import DeferJobCreatedManager
-from awx.main.fields import JSONField
 from awx.main.constants import MINIMAL_EVENTS
 from awx.main.models.base import CreatedModifiedModel
 from awx.main.utils import ignore_inventory_computed_fields, camelcase_to_underscore
@@ -209,10 +209,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
         max_length=100,
         choices=EVENT_CHOICES,
     )
-    event_data = JSONField(
-        blank=True,
-        default=dict,
-    )
+    event_data = JSONBlob(default=dict, blank=True)
     failed = models.BooleanField(
         default=False,
         editable=False,
@@ -396,7 +393,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
                     connection.on_commit(_send_notifications)
 
         for field in ('playbook', 'play', 'task', 'role'):
-            value = force_text(event_data.get(field, '')).strip()
+            value = force_str(event_data.get(field, '')).strip()
             if value != getattr(self, field):
                 setattr(self, field, value)
         if settings.LOG_AGGREGATOR_ENABLED:
@@ -648,10 +645,7 @@ class BaseCommandEvent(CreatedModifiedModel):
     class Meta:
         abstract = True
 
-    event_data = JSONField(
-        blank=True,
-        default=dict,
-    )
+    event_data = JSONBlob(default=dict, blank=True)
     uuid = models.CharField(
         max_length=1024,
         default='',

awx/main/models/execution_environments.py

@@ -1,5 +1,5 @@
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from awx.api.versioning import reverse
 from awx.main.models.base import CommonModel

awx/main/models/ha.py

@@ -9,7 +9,7 @@ from django.core.validators import MinValueValidator
 from django.db import models, connection
 from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 from django.utils.timezone import now, timedelta
 
@@ -19,7 +19,6 @@ from solo.models import SingletonModel
 from awx import __version__ as awx_application_version
 from awx.api.versioning import reverse
 from awx.main.managers import InstanceManager, InstanceGroupManager, UUID_DEFAULT
-from awx.main.fields import JSONField
 from awx.main.constants import JOB_FOLDER_PREFIX
 from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
 from awx.main.models.unified_jobs import UnifiedJob
@@ -233,13 +232,19 @@ class Instance(HasPolicyEditsMixin, BaseModel):
 
     def refresh_capacity_fields(self):
         """Update derived capacity fields from cpu and memory (no save)"""
-        self.cpu_capacity = get_cpu_effective_capacity(self.cpu)
-        self.mem_capacity = get_mem_effective_capacity(self.memory)
+        if self.node_type == 'hop':
+            self.cpu_capacity = 0
+            self.mem_capacity = 0  # formula has a non-zero offset, so we make sure it is 0 for hop nodes
+        else:
+            self.cpu_capacity = get_cpu_effective_capacity(self.cpu)
+            self.mem_capacity = get_mem_effective_capacity(self.memory)
         self.set_capacity_value()
 
-    def save_health_data(self, version, cpu, memory, uuid=None, update_last_seen=False, errors=''):
-        self.last_health_check = now()
-        update_fields = ['last_health_check']
+    def save_health_data(self, version=None, cpu=0, memory=0, uuid=None, update_last_seen=False, errors=''):
+        update_fields = ['errors']
+        if self.node_type != 'hop':
+            self.last_health_check = now()
+            update_fields.append('last_health_check')
 
         if update_last_seen:
             self.last_seen = self.last_health_check
@@ -247,11 +252,11 @@ class Instance(HasPolicyEditsMixin, BaseModel):
 
         if uuid is not None and self.uuid != uuid:
             if self.uuid is not None:
-                logger.warn(f'Self-reported uuid of {self.hostname} changed from {self.uuid} to {uuid}')
+                logger.warning(f'Self-reported uuid of {self.hostname} changed from {self.uuid} to {uuid}')
             self.uuid = uuid
             update_fields.append('uuid')
 
-        if self.version != version:
+        if version is not None and self.version != version:
             self.version = version
             update_fields.append('version')
 
@@ -270,7 +275,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
             self.errors = ''
         else:
             self.mark_offline(perform_save=False, errors=errors)
-        update_fields.extend(['cpu_capacity', 'mem_capacity', 'capacity', 'errors'])
+        update_fields.extend(['cpu_capacity', 'mem_capacity', 'capacity'])
 
         # disabling activity stream will avoid extra queries, which is important for heatbeat actions
         from awx.main.signals import disable_activity_stream
@@ -322,8 +327,8 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
     )
     policy_instance_percentage = models.IntegerField(default=0, help_text=_("Percentage of Instances to automatically assign to this group"))
     policy_instance_minimum = models.IntegerField(default=0, help_text=_("Static minimum number of Instances to automatically assign to this group"))
-    policy_instance_list = JSONField(
-        default=[], blank=True, help_text=_("List of exact-match Instances that will always be automatically assigned to this group")
+    policy_instance_list = models.JSONField(
+        default=list, blank=True, help_text=_("List of exact-match Instances that will always be automatically assigned to this group")
     )
 
     POLICY_FIELDS = frozenset(('policy_instance_list', 'policy_instance_minimum', 'policy_instance_percentage'))

awx/main/models/inventory.py

@@ -14,7 +14,7 @@ import yaml
 # Django
 from django.conf import settings
 from django.db import models, connection
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.db import transaction
 from django.core.exceptions import ValidationError
 from django.utils.timezone import now
@@ -29,7 +29,6 @@ from awx.main.constants import CLOUD_PROVIDERS
 from awx.main.consumers import emit_channel_notification
 from awx.main.fields import (
     ImplicitRoleField,
-    JSONBField,
     SmartFilterField,
     OrderedManyToManyField,
 )
@@ -488,7 +487,7 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
         editable=False,
         help_text=_('Inventory source(s) that created or modified this host.'),
     )
-    ansible_facts = JSONBField(
+    ansible_facts = models.JSONField(
         blank=True,
         default=dict,
         help_text=_('Arbitrary JSON structure of most recent ansible_facts, per-host.'),

awx/main/models/jobs.py

@@ -19,7 +19,7 @@ from django.db import models
 # from django.core.cache import cache
 from django.utils.encoding import smart_str
 from django.utils.timezone import now
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import FieldDoesNotExist
 
 # REST Framework
@@ -44,7 +44,7 @@ from awx.main.models.notifications import (
     JobNotificationMixin,
 )
 from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField
-from awx.main.fields import ImplicitRoleField, JSONField, AskForField
+from awx.main.fields import ImplicitRoleField, AskForField
 from awx.main.models.mixins import (
     ResourceMixin,
     SurveyJobTemplateMixin,
@@ -546,9 +546,10 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
         editable=False,
         through='JobHostSummary',
     )
-    artifacts = JSONField(
-        blank=True,
+    artifacts = models.JSONField(
         default=dict,
+        null=True,
+        blank=True,
         editable=False,
     )
     scm_revision = models.CharField(
@@ -885,7 +886,7 @@ class LaunchTimeConfigBase(BaseModel):
     )
     # All standard fields are stored in this dictionary field
     # This is a solution to the nullable CharField problem, specific to prompting
-    char_prompts = JSONField(blank=True, default=dict)
+    char_prompts = models.JSONField(default=dict, null=True, blank=True)
 
     def prompts_dict(self, display=False):
         data = {}
@@ -938,12 +939,13 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
         abstract = True
 
     # Special case prompting fields, even more special than the other ones
-    extra_data = JSONField(blank=True, default=dict)
+    extra_data = models.JSONField(default=dict, null=True, blank=True)
     survey_passwords = prevent_search(
-        JSONField(
-            blank=True,
+        models.JSONField(
             default=dict,
             editable=False,
+            null=True,
+            blank=True,
         )
     )
     # Credentials needed for non-unified job / unified JT models

awx/main/models/label.py

@@ -3,7 +3,7 @@
 
 # Django
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 # AWX
 from awx.api.versioning import reverse

awx/main/models/mixins.py

@@ -15,7 +15,7 @@ from django.core.exceptions import ValidationError
 from django.db import models
 from django.db.models.query import QuerySet
 from django.utils.crypto import get_random_string
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 # AWX
 from awx.main.models.base import prevent_search
@@ -24,7 +24,7 @@ from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_lice
 from awx.main.utils.execution_environments import get_default_execution_environment
 from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
 from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
-from awx.main.fields import JSONField, AskForField
+from awx.main.fields import AskForField
 from awx.main.constants import ACTIVE_STATES
 
 
@@ -103,12 +103,7 @@ class SurveyJobTemplateMixin(models.Model):
     survey_enabled = models.BooleanField(
         default=False,
     )
-    survey_spec = prevent_search(
-        JSONField(
-            blank=True,
-            default=dict,
-        )
-    )
+    survey_spec = prevent_search(models.JSONField(default=dict, blank=True))
     ask_variables_on_launch = AskForField(blank=True, default=False, allows_field='extra_vars')
 
     def survey_password_variables(self):
@@ -370,10 +365,11 @@ class SurveyJobMixin(models.Model):
         abstract = True
 
     survey_passwords = prevent_search(
-        JSONField(
-            blank=True,
+        models.JSONField(
             default=dict,
             editable=False,
+            null=True,
+            blank=True,
         )
     )
 

awx/main/models/notifications.py

@@ -10,8 +10,8 @@ from django.db import models
 from django.conf import settings
 from django.core.mail.message import EmailMessage
 from django.db import connection
-from django.utils.translation import ugettext_lazy as _
-from django.utils.encoding import smart_str, force_text
+from django.utils.translation import gettext_lazy as _
+from django.utils.encoding import smart_str, force_str
 from jinja2 import sandbox, ChainableUndefined
 from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
 
@@ -28,7 +28,6 @@ from awx.main.notifications.mattermost_backend import MattermostBackend
 from awx.main.notifications.grafana_backend import GrafanaBackend
 from awx.main.notifications.rocketchat_backend import RocketChatBackend
 from awx.main.notifications.irc_backend import IrcBackend
-from awx.main.fields import JSONField
 
 
 logger = logging.getLogger('awx.main.models.notifications')
@@ -70,12 +69,12 @@ class NotificationTemplate(CommonModelNameNotUnique):
         choices=NOTIFICATION_TYPE_CHOICES,
     )
 
-    notification_configuration = prevent_search(JSONField(blank=False))
+    notification_configuration = prevent_search(models.JSONField(default=dict))
 
     def default_messages():
         return {'started': None, 'success': None, 'error': None, 'workflow_approval': None}
 
-    messages = JSONField(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.'))
+    messages = models.JSONField(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.'))
 
     def has_message(self, condition):
         potential_template = self.messages.get(condition, {})
@@ -187,7 +186,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
     def display_notification_configuration(self):
         field_val = self.notification_configuration.copy()
         for field in self.notification_class.init_parameters:
-            if field in field_val and force_text(field_val[field]).startswith('$encrypted$'):
+            if field in field_val and force_str(field_val[field]).startswith('$encrypted$'):
                 field_val[field] = '$encrypted$'
         return field_val
 
@@ -237,7 +236,7 @@ class Notification(CreatedModifiedModel):
         default='',
         editable=False,
     )
-    body = JSONField(blank=True)
+    body = models.JSONField(default=dict, null=True, blank=True)
 
     def get_absolute_url(self, request=None):
         return reverse('api:notification_detail', kwargs={'pk': self.pk}, request=request)
@@ -515,7 +514,7 @@ class JobNotificationMixin(object):
         try:
             notification_templates = self.get_notification_templates()
         except Exception:
-            logger.warn("No notification template defined for emitting notification")
+            logger.warning("No notification template defined for emitting notification")
             return
 
         if not notification_templates:

awx/main/models/oauth.py

@@ -6,7 +6,7 @@ import re
 from django.core.validators import RegexValidator
 from django.db import models, connection
 from django.utils.timezone import now
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 
 # Django OAuth Toolkit

awx/main/models/organization.py

@@ -8,7 +8,7 @@ from django.db import models
 from django.contrib.auth.models import User
 from django.contrib.sessions.models import Session
 from django.utils.timezone import now as tz_now
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 
 # AWX

awx/main/models/projects.py

@@ -9,8 +9,8 @@ import urllib.parse as urlparse
 # Django
 from django.conf import settings
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
-from django.utils.encoding import smart_str, smart_text
+from django.utils.translation import gettext_lazy as _
+from django.utils.encoding import smart_str
 from django.utils.text import slugify
 from django.core.exceptions import ValidationError
 from django.utils.timezone import now, make_aware, get_default_timezone
@@ -38,7 +38,6 @@ from awx.main.models.rbac import (
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
     ROLE_SINGLETON_SYSTEM_AUDITOR,
 )
-from awx.main.fields import JSONField
 
 __all__ = ['Project', 'ProjectUpdate']
 
@@ -214,7 +213,7 @@ class ProjectOptions(models.Model):
                 for filename in filenames:
                     playbook = could_be_playbook(project_path, dirpath, filename)
                     if playbook is not None:
-                        results.append(smart_text(playbook))
+                        results.append(smart_str(playbook))
         return sorted(results, key=lambda x: smart_str(x).lower())
 
     @property
@@ -230,7 +229,7 @@ class ProjectOptions(models.Model):
                 for filename in filenames:
                     inv_path = could_be_inventory(project_path, dirpath, filename)
                     if inv_path is not None:
-                        results.append(smart_text(inv_path))
+                        results.append(smart_str(inv_path))
                         if len(results) > max_inventory_listing:
                             break
                 if len(results) > max_inventory_listing:
@@ -294,17 +293,17 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         help_text=_('The last revision fetched by a project update'),
     )
 
-    playbook_files = JSONField(
+    playbook_files = models.JSONField(
+        default=list,
         blank=True,
-        default=[],
         editable=False,
         verbose_name=_('Playbook Files'),
         help_text=_('List of playbooks found in the project'),
     )
 
-    inventory_files = JSONField(
+    inventory_files = models.JSONField(
+        default=list,
         blank=True,
-        default=[],
         editable=False,
         verbose_name=_('Inventory Files'),
         help_text=_('Suggested list of content that could be Ansible inventory in the project'),

awx/main/models/rbac.py

@@ -11,7 +11,7 @@ import re
 from django.db import models, transaction, connection
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.contenttypes.fields import GenericForeignKey
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 # AWX
 from awx.api.versioning import reverse

awx/main/models/schedules.py

@@ -14,7 +14,7 @@ from dateutil.zoneinfo import get_zonefile_instance
 from django.db import models
 from django.db.models.query import QuerySet
 from django.utils.timezone import now, make_aware
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 # AWX
 from awx.api.versioning import reverse
@@ -103,7 +103,7 @@ class Schedule(PrimordialModel, LaunchTimeConfig):
                     for zone in all_zones:
                         if fname.endswith(zone):
                             return zone
-        logger.warn('Could not detect valid zoneinfo for {}'.format(self.rrule))
+        logger.warning('Could not detect valid zoneinfo for {}'.format(self.rrule))
         return ''
 
     @property

awx/main/models/unified_jobs.py

@@ -19,9 +19,9 @@ from collections import OrderedDict
 from django.conf import settings
 from django.db import models, connection
 from django.core.exceptions import NON_FIELD_ERRORS
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.utils.timezone import now
-from django.utils.encoding import smart_text
+from django.utils.encoding import smart_str
 from django.contrib.contenttypes.models import ContentType
 
 # REST Framework
@@ -54,7 +54,7 @@ from awx.main.utils import polymorphic
 from awx.main.constants import ACTIVE_STATES, CAN_CANCEL
 from awx.main.redact import UriCleaner, REPLACE_STR
 from awx.main.consumers import emit_channel_notification
-from awx.main.fields import JSONField, JSONBField, AskForField, OrderedManyToManyField
+from awx.main.fields import AskForField, OrderedManyToManyField
 
 __all__ = ['UnifiedJobTemplate', 'UnifiedJob', 'StdoutMaxBytesExceeded']
 
@@ -357,7 +357,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
         validated_kwargs = kwargs.copy()
         if unallowed_fields:
             if parent_field_name is None:
-                logger.warn('Fields {} are not allowed as overrides to spawn from {}.'.format(', '.join(unallowed_fields), self))
+                logger.warning('Fields {} are not allowed as overrides to spawn from {}.'.format(', '.join(unallowed_fields), self))
             for f in unallowed_fields:
                 validated_kwargs.pop(f)
 
@@ -653,9 +653,10 @@ class UnifiedJob(
         editable=False,
     )
     job_env = prevent_search(
-        JSONField(
-            blank=True,
+        models.JSONField(
             default=dict,
+            null=True,
+            blank=True,
             editable=False,
         )
     )
@@ -704,7 +705,7 @@ class UnifiedJob(
         'Credential',
         related_name='%(class)ss',
     )
-    installed_collections = JSONBField(
+    installed_collections = models.JSONField(
         blank=True,
         default=dict,
         editable=False,
@@ -1090,7 +1091,7 @@ class UnifiedJob(
                 # function assume a str-based fd will be returned; decode
                 # .write() calls on the fly to maintain this interface
                 _write = fd.write
-                fd.write = lambda s: _write(smart_text(s))
+                fd.write = lambda s: _write(smart_str(s))
                 tbl = self._meta.db_table + 'event'
                 created_by_cond = ''
                 if self.has_unpartitioned_events:
@@ -1205,7 +1206,7 @@ class UnifiedJob(
             try:
                 extra_data_dict = parse_yaml_or_json(extra_data, silent_failure=False)
             except Exception as e:
-                logger.warn("Exception deserializing extra vars: " + str(e))
+                logger.warning("Exception deserializing extra vars: " + str(e))
             evars = self.extra_vars_dict
             evars.update(extra_data_dict)
             self.update_fields(extra_vars=json.dumps(evars))
@@ -1273,7 +1274,7 @@ class UnifiedJob(
             id=self.id,
             name=self.name,
             url=self.get_ui_url(),
-            created_by=smart_text(self.created_by),
+            created_by=smart_str(self.created_by),
             started=self.started.isoformat() if self.started is not None else None,
             finished=self.finished.isoformat() if self.finished is not None else None,
             status=self.status,

awx/main/models/workflow.py

@@ -11,7 +11,7 @@ from urllib.parse import urljoin
 # Django
 from django.db import connection, models
 from django.conf import settings
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import ObjectDoesNotExist
 
 # from django import settings as tower_settings
@@ -40,7 +40,6 @@ from awx.main.models.mixins import (
 from awx.main.models.jobs import LaunchTimeConfigBase, LaunchTimeConfig, JobTemplate
 from awx.main.models.credential import Credential
 from awx.main.redact import REPLACE_STR
-from awx.main.fields import JSONField
 from awx.main.utils import schedule_task_manager
 
 
@@ -232,9 +231,10 @@ class WorkflowJobNode(WorkflowNodeBase):
         default=None,
         on_delete=models.CASCADE,
     )
-    ancestor_artifacts = JSONField(
-        blank=True,
+    ancestor_artifacts = models.JSONField(
         default=dict,
+        null=True,
+        blank=True,
         editable=False,
     )
     do_not_run = models.BooleanField(

awx/main/notifications/grafana_backend.py

@@ -7,8 +7,8 @@ import logging
 import requests
 import dateutil.parser as dp
 
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
 
 from awx.main.notifications.base import AWXBaseEmailBackend
 from awx.main.notifications.custom_notification_base import CustomNotificationBase
@@ -82,9 +82,9 @@ class GrafanaBackend(AWXBaseEmailBackend, CustomNotificationBase):
                     if m.body.get('finished'):
                         grafana_data['timeEnd'] = int((dp.parse(m.body['finished']).replace(tzinfo=None) - epoch).total_seconds() * 1000)
                 except ValueError:
-                    logger.error(smart_text(_("Error converting time {} or timeEnd {} to int.").format(m.body['started'], m.body['finished'])))
+                    logger.error(smart_str(_("Error converting time {} or timeEnd {} to int.").format(m.body['started'], m.body['finished'])))
                     if not self.fail_silently:
-                        raise Exception(smart_text(_("Error converting time {} and/or timeEnd {} to int.").format(m.body['started'], m.body['finished'])))
+                        raise Exception(smart_str(_("Error converting time {} and/or timeEnd {} to int.").format(m.body['started'], m.body['finished'])))
             grafana_data['isRegion'] = self.isRegion
             grafana_data['dashboardId'] = self.dashboardId
             grafana_data['panelId'] = self.panelId
@@ -97,8 +97,8 @@ class GrafanaBackend(AWXBaseEmailBackend, CustomNotificationBase):
                 "{}/api/annotations".format(m.recipients()[0]), json=grafana_data, headers=grafana_headers, verify=(not self.grafana_no_verify_ssl)
             )
             if r.status_code >= 400:
-                logger.error(smart_text(_("Error sending notification grafana: {}").format(r.status_code)))
+                logger.error(smart_str(_("Error sending notification grafana: {}").format(r.status_code)))
                 if not self.fail_silently:
-                    raise Exception(smart_text(_("Error sending notification grafana: {}").format(r.status_code)))
+                    raise Exception(smart_str(_("Error sending notification grafana: {}").format(r.status_code)))
             sent_messages += 1
         return sent_messages

awx/main/notifications/irc_backend.py

@@ -7,8 +7,8 @@ import logging
 
 import irc.client
 
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
 
 from awx.main.notifications.base import AWXBaseEmailBackend
 from awx.main.notifications.custom_notification_base import CustomNotificationBase
@@ -55,7 +55,7 @@ class IrcBackend(AWXBaseEmailBackend, CustomNotificationBase):
                 connect_factory=connection_factory,
             )
         except irc.client.ServerConnectionError as e:
-            logger.error(smart_text(_("Exception connecting to irc server: {}").format(e)))
+            logger.error(smart_str(_("Exception connecting to irc server: {}").format(e)))
             if not self.fail_silently:
                 raise
         return True

awx/main/notifications/mattermost_backend.py

@@ -4,8 +4,8 @@
 import logging
 import requests
 
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
 
 from awx.main.notifications.base import AWXBaseEmailBackend
 from awx.main.notifications.custom_notification_base import CustomNotificationBase
@@ -44,8 +44,8 @@ class MattermostBackend(AWXBaseEmailBackend, CustomNotificationBase):
 
             r = requests.post("{}".format(m.recipients()[0]), json=payload, verify=(not self.mattermost_no_verify_ssl))
             if r.status_code >= 400:
-                logger.error(smart_text(_("Error sending notification mattermost: {}").format(r.status_code)))
+                logger.error(smart_str(_("Error sending notification mattermost: {}").format(r.status_code)))
                 if not self.fail_silently:
-                    raise Exception(smart_text(_("Error sending notification mattermost: {}").format(r.status_code)))
+                    raise Exception(smart_str(_("Error sending notification mattermost: {}").format(r.status_code)))
             sent_messages += 1
         return sent_messages

awx/main/notifications/pagerduty_backend.py

@@ -5,8 +5,8 @@ import json
 import logging
 import pygerduty
 
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
 
 from awx.main.notifications.base import AWXBaseEmailBackend
 from awx.main.notifications.custom_notification_base import CustomNotificationBase
@@ -78,13 +78,13 @@ class PagerDutyBackend(AWXBaseEmailBackend, CustomNotificationBase):
         except Exception as e:
             if not self.fail_silently:
                 raise
-            logger.error(smart_text(_("Exception connecting to PagerDuty: {}").format(e)))
+            logger.error(smart_str(_("Exception connecting to PagerDuty: {}").format(e)))
         for m in messages:
             try:
                 pager.trigger_incident(m.recipients()[0], description=m.subject, details=m.body, client=m.from_email)
                 sent_messages += 1
             except Exception as e:
-                logger.error(smart_text(_("Exception sending messages: {}").format(e)))
+                logger.error(smart_str(_("Exception sending messages: {}").format(e)))
                 if not self.fail_silently:
                     raise
         return sent_messages

awx/main/notifications/rocketchat_backend.py

@@ -5,8 +5,8 @@ import logging
 import requests
 import json
 
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
 
 from awx.main.notifications.base import AWXBaseEmailBackend
 from awx.main.utils import get_awx_http_client_headers
@@ -44,8 +44,8 @@ class RocketChatBackend(AWXBaseEmailBackend, CustomNotificationBase):
             )
 
             if r.status_code >= 400:
-                logger.error(smart_text(_("Error sending notification rocket.chat: {}").format(r.status_code)))
+                logger.error(smart_str(_("Error sending notification rocket.chat: {}").format(r.status_code)))
                 if not self.fail_silently:
-                    raise Exception(smart_text(_("Error sending notification rocket.chat: {}").format(r.status_code)))
+                    raise Exception(smart_str(_("Error sending notification rocket.chat: {}").format(r.status_code)))
             sent_messages += 1
         return sent_messages

awx/main/notifications/slack_backend.py

@@ -5,8 +5,8 @@ import logging
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
 
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
 
 from awx.main.notifications.base import AWXBaseEmailBackend
 from awx.main.notifications.custom_notification_base import CustomNotificationBase
@@ -53,7 +53,7 @@ class SlackBackend(AWXBaseEmailBackend, CustomNotificationBase):
                     else:
                         raise RuntimeError("Slack Notification unable to send {}: {} ({})".format(r, m.subject, response['error']))
             except SlackApiError as e:
-                logger.error(smart_text(_("Exception sending messages: {}").format(e)))
+                logger.error(smart_str(_("Exception sending messages: {}").format(e)))
                 if not self.fail_silently:
                     raise
         return sent_messages

awx/main/notifications/twilio_backend.py

@@ -5,8 +5,8 @@ import logging
 
 from twilio.rest import Client
 
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
 
 from awx.main.notifications.base import AWXBaseEmailBackend
 from awx.main.notifications.custom_notification_base import CustomNotificationBase
@@ -37,14 +37,14 @@ class TwilioBackend(AWXBaseEmailBackend, CustomNotificationBase):
         except Exception as e:
             if not self.fail_silently:
                 raise
-            logger.error(smart_text(_("Exception connecting to Twilio: {}").format(e)))
+            logger.error(smart_str(_("Exception connecting to Twilio: {}").format(e)))
 
         for m in messages:
             try:
                 connection.messages.create(to=m.to, from_=m.from_email, body=m.subject)
                 sent_messages += 1
             except Exception as e:
-                logger.error(smart_text(_("Exception sending messages: {}").format(e)))
+                logger.error(smart_str(_("Exception sending messages: {}").format(e)))
                 if not self.fail_silently:
                     raise
         return sent_messages

awx/main/notifications/webhook_backend.py

@@ -5,8 +5,8 @@ import json
 import logging
 import requests
 
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
 
 from awx.main.notifications.base import AWXBaseEmailBackend
 from awx.main.utils import get_awx_http_client_headers
@@ -76,8 +76,8 @@ class WebhookBackend(AWXBaseEmailBackend, CustomNotificationBase):
                 verify=(not self.disable_ssl_verification),
             )
             if r.status_code >= 400:
-                logger.error(smart_text(_("Error sending notification webhook: {}").format(r.status_code)))
+                logger.error(smart_str(_("Error sending notification webhook: {}").format(r.status_code)))
                 if not self.fail_silently:
-                    raise Exception(smart_text(_("Error sending notification webhook: {}").format(r.status_code)))
+                    raise Exception(smart_str(_("Error sending notification webhook: {}").format(r.status_code)))
             sent_messages += 1
         return sent_messages

awx/main/registrar.py

@@ -32,7 +32,7 @@ class ActivityStreamRegistrar(object):
             post_save.disconnect(dispatch_uid=str(self.__class__) + str(model) + "_create")
             pre_save.disconnect(dispatch_uid=str(self.__class__) + str(model) + "_update")
             pre_delete.disconnect(dispatch_uid=str(self.__class__) + str(model) + "_delete")
-            self.models.pop(model)
+            self.models.remove(model)
 
             for m2mfield in model._meta.many_to_many:
                 m2m_attr = getattr(model, m2mfield.name)

awx/main/routing.py

@@ -1,8 +1,8 @@
 import redis
 import logging
 
-from django.conf.urls import url
 from django.conf import settings
+from django.urls import re_path
 
 from channels.auth import AuthMiddlewareStack
 from channels.routing import ProtocolTypeRouter, URLRouter
@@ -21,14 +21,14 @@ class AWXProtocolTypeRouter(ProtocolTypeRouter):
                 logger.debug(f"cleaning up Redis key {k}")
                 r.delete(k)
         except redis.exceptions.RedisError as e:
-            logger.warn("encountered an error communicating with redis.")
+            logger.warning("encountered an error communicating with redis.")
             raise e
         super().__init__(*args, **kwargs)
 
 
 websocket_urlpatterns = [
-    url(r'websocket/$', consumers.EventConsumer),
-    url(r'websocket/broadcast/$', consumers.BroadcastConsumer),
+    re_path(r'websocket/$', consumers.EventConsumer),
+    re_path(r'websocket/broadcast/$', consumers.BroadcastConsumer),
 ]
 
 application = AWXProtocolTypeRouter(

awx/main/scheduler/dag_workflow.py

@@ -1,5 +1,5 @@
-from django.utils.translation import ugettext_lazy as _
-from django.utils.encoding import smart_text
+from django.utils.translation import gettext_lazy as _
+from django.utils.encoding import smart_str
 
 # Python
 from awx.main.models import (
@@ -171,7 +171,7 @@ class WorkflowDAG(SimpleDAG):
                 parms['node_status'] = ",".join(["({},{})".format(id, status) for id, status in failed_path_nodes_id_status])
             if len(failed_unified_job_template_node_ids) > 0:
                 parms['no_ufjt'] = ",".join(failed_unified_job_template_node_ids)
-            return True, smart_text(s.format(**parms))
+            return True, smart_str(s.format(**parms))
         return False, None
 
     r'''

awx/main/scheduler/kubernetes.py

@@ -7,7 +7,7 @@ from urllib import parse as urlparse
 from django.conf import settings
 from kubernetes import client, config
 from django.utils.functional import cached_property
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from awx.main.utils.common import parse_yaml_or_json, deepmerge
 from awx.main.utils.execution_environments import get_default_pod_spec

awx/main/scheduler/task_manager.py

@@ -10,7 +10,7 @@ from types import SimpleNamespace
 
 # Django
 from django.db import transaction, connection
-from django.utils.translation import ugettext_lazy as _, gettext_noop
+from django.utils.translation import gettext_lazy as _, gettext_noop
 from django.utils.timezone import now as tz_now
 from django.conf import settings
 
@@ -493,6 +493,8 @@ class TaskManager:
                 control_instance.jobs_running += 1
                 self.dependency_graph.add_job(task)
                 execution_instance = self.real_instances[control_instance.hostname]
+                task.log_lifecycle("controller_node_chosen")
+                task.log_lifecycle("execution_node_chosen")
                 self.start_task(task, self.controlplane_ig, task.get_jobs_fail_chain(), execution_instance)
                 found_acceptable_queue = True
                 continue
@@ -572,7 +574,7 @@ class TaskManager:
                 timeout_message = _("The approval node {name} ({pk}) has expired after {timeout} seconds.").format(
                     name=task.name, pk=task.pk, timeout=task.timeout
                 )
-                logger.warn(timeout_message)
+                logger.warning(timeout_message)
                 task.timed_out = True
                 task.status = 'failed'
                 task.send_approval_notification('timed_out')

awx/main/tasks/__init__.py

@@ -0,0 +1 @@
+from . import jobs, receptor, system  # noqa

awx/main/tasks/callback.py

@@ -8,7 +8,7 @@ import stat
 # Django
 from django.utils.timezone import now
 from django.conf import settings
-from django_guid.middleware import GuidMiddleware
+from django_guid import get_guid
 
 # AWX
 from awx.main.redact import UriCleaner
@@ -25,7 +25,7 @@ class RunnerCallback:
     def __init__(self, model=None):
         self.parent_workflow_job_id = None
         self.host_map = {}
-        self.guid = GuidMiddleware.get_guid()
+        self.guid = get_guid()
         self.job_created = None
         self.recent_event_timings = deque(maxlen=settings.MAX_WEBSOCKET_EVENT_RATE)
         self.dispatcher = CallbackQueueDispatcher()
@@ -154,7 +154,7 @@ class RunnerCallback:
         if self.instance.cancel_flag or self.instance.status == 'canceled':
             cancel_wait = (now() - self.instance.modified).seconds if self.instance.modified else 0
             if cancel_wait > 5:
-                logger.warn('Request to cancel {} took {} seconds to complete.'.format(self.instance.log_format, cancel_wait))
+                logger.warning('Request to cancel {} took {} seconds to complete.'.format(self.instance.log_format, cancel_wait))
             return True
         return False
 

awx/main/tasks/jobs.py

@@ -40,6 +40,7 @@ from awx.main.constants import (
     STANDARD_INVENTORY_UPDATE_ENV,
     JOB_FOLDER_PREFIX,
     MAX_ISOLATED_PATH_COLON_DELIMITER,
+    CONTAINER_VOLUMES_MOUNT_TYPES,
 )
 from awx.main.models import (
     Instance,
@@ -80,7 +81,7 @@ from awx.main.utils.handlers import SpecialInventoryHandler
 from awx.main.tasks.system import handle_success_and_failure_notifications, update_smart_memberships_for_inventory, update_inventory_computed_fields
 from awx.main.utils.update_model import update_model
 from rest_framework.exceptions import PermissionDenied
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 logger = logging.getLogger('awx.main.tasks.jobs')
 
@@ -163,8 +164,14 @@ class BaseTask(object):
                 # Using z allows the dir to be mounted by multiple containers
                 # Uppercase Z restricts access (in weird ways) to 1 container at a time
                 if this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER:
-                    src, dest, scontext = this_path.split(':')
-                    params['container_volume_mounts'].append(f'{src}:{dest}:{scontext}')
+                    src, dest, mount_option = this_path.split(':')
+
+                    # mount_option validation via performed via API, but since this can be overriden via settings.py
+                    if mount_option not in CONTAINER_VOLUMES_MOUNT_TYPES:
+                        mount_option = 'z'
+                        logger.warning(f'The path {this_path} has volume mount type {mount_option} which is not supported. Using "z" instead.')
+
+                    params['container_volume_mounts'].append(f'{src}:{dest}:{mount_option}')
                 elif this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER - 1:
                     src, dest = this_path.split(':')
                     params['container_volume_mounts'].append(f'{src}:{dest}:z')
@@ -816,11 +823,12 @@ class RunJob(BaseTask):
         return job.playbook
 
     def build_extra_vars_file(self, job, private_data_dir):
-        # Define special extra_vars for AWX, combine with job.extra_vars.
-        extra_vars = job.awx_meta_vars()
-
+        extra_vars = dict()
+        # load in JT extra vars
         if job.extra_vars_dict:
             extra_vars.update(json.loads(job.decrypted_extra_vars()))
+        # load in meta vars, overriding any variable set in JT extra vars
+        extra_vars.update(job.awx_meta_vars())
 
         # By default, all extra vars disallow Jinja2 template usage for
         # security reasons; top level key-values defined in JT.extra_vars, however,
@@ -854,24 +862,6 @@ class RunJob(BaseTask):
                 d[r'Vault password \({}\):\s*?$'.format(vault_id)] = k
         return d
 
-    def build_execution_environment_params(self, instance, private_data_dir):
-        if settings.IS_K8S:
-            return {}
-
-        params = super(RunJob, self).build_execution_environment_params(instance, private_data_dir)
-        # If this has an insights agent and it is not already mounted then show it
-        insights_dir = os.path.dirname(settings.INSIGHTS_SYSTEM_ID_FILE)
-        if instance.use_fact_cache and os.path.exists(insights_dir):
-            logger.info('not parent of others')
-            params.setdefault('container_volume_mounts', [])
-            params['container_volume_mounts'].extend(
-                [
-                    f"{insights_dir}:{insights_dir}:Z",
-                ]
-            )
-
-        return params
-
     def pre_run_hook(self, job, private_data_dir):
         super(RunJob, self).pre_run_hook(job, private_data_dir)
         if job.inventory is None:
@@ -1896,14 +1886,6 @@ class RunAdHocCommand(BaseTask):
         if ad_hoc_command.verbosity:
             args.append('-%s' % ('v' * min(5, ad_hoc_command.verbosity)))
 
-        extra_vars = ad_hoc_command.awx_meta_vars()
-
-        if ad_hoc_command.extra_vars_dict:
-            redacted_extra_vars, removed_vars = extract_ansible_vars(ad_hoc_command.extra_vars_dict)
-            if removed_vars:
-                raise ValueError(_("{} are prohibited from use in ad hoc commands.").format(", ".join(removed_vars)))
-            extra_vars.update(ad_hoc_command.extra_vars_dict)
-
         if ad_hoc_command.limit:
             args.append(ad_hoc_command.limit)
         else:
@@ -1912,13 +1894,13 @@ class RunAdHocCommand(BaseTask):
         return args
 
     def build_extra_vars_file(self, ad_hoc_command, private_data_dir):
-        extra_vars = ad_hoc_command.awx_meta_vars()
-
+        extra_vars = dict()
         if ad_hoc_command.extra_vars_dict:
             redacted_extra_vars, removed_vars = extract_ansible_vars(ad_hoc_command.extra_vars_dict)
             if removed_vars:
                 raise ValueError(_("{} are prohibited from use in ad hoc commands.").format(", ".join(removed_vars)))
             extra_vars.update(ad_hoc_command.extra_vars_dict)
+        extra_vars.update(ad_hoc_command.awx_meta_vars())
         self._write_extra_vars_file(private_data_dir, extra_vars)
 
     def build_module_name(self, ad_hoc_command):

awx/main/tasks/receptor.py

@@ -7,8 +7,6 @@ import logging
 import os
 import shutil
 import socket
-import sys
-import threading
 import time
 import yaml
 
@@ -26,6 +24,8 @@ from awx.main.utils.common import (
     parse_yaml_or_json,
     cleanup_new_process,
 )
+from awx.main.constants import MAX_ISOLATED_PATH_COLON_DELIMITER
+
 
 # Receptorctl
 from receptorctl.socket_interface import ReceptorControl
@@ -164,7 +164,7 @@ def run_until_complete(node, timing_data=None, **kwargs):
         if settings.RECEPTOR_RELEASE_WORK:
             res = receptor_ctl.simple_command(f"work release {unit_id}")
             if res != {'released': unit_id}:
-                logger.warn(f'Could not confirm release of receptor work unit id {unit_id} from {node}, data: {res}')
+                logger.warning(f'Could not confirm release of receptor work unit id {unit_id} from {node}, data: {res}')
 
         receptor_ctl.close()
 
@@ -247,16 +247,6 @@ def worker_cleanup(node_name, vargs, timeout=300.0):
     return stdout
 
 
-class TransmitterThread(threading.Thread):
-    def run(self):
-        self.exc = None
-
-        try:
-            super().run()
-        except Exception:
-            self.exc = sys.exc_info()
-
-
 class AWXReceptorJob:
     def __init__(self, task, runner_params=None):
         self.task = task
@@ -296,46 +286,47 @@ class AWXReceptorJob:
         # reading.
         sockin, sockout = socket.socketpair()
 
-        transmitter_thread = TransmitterThread(target=self.transmit, args=[sockin])
-        transmitter_thread.start()
-
-        # submit our work, passing
-        # in the right side of our socketpair for reading.
-        _kw = {}
+        # Prepare the submit_work kwargs before creating threads, because references to settings are not thread-safe
+        work_submit_kw = dict(worktype=self.work_type, params=self.receptor_params, signwork=self.sign_work)
         if self.work_type == 'ansible-runner':
-            _kw['node'] = self.task.instance.execution_node
-            use_stream_tls = get_conn_type(_kw['node'], receptor_ctl).name == "STREAMTLS"
-            _kw['tlsclient'] = get_tls_client(use_stream_tls)
-        result = receptor_ctl.submit_work(worktype=self.work_type, payload=sockout.makefile('rb'), params=self.receptor_params, signwork=self.sign_work, **_kw)
-        self.unit_id = result['unitid']
-        # Update the job with the work unit in-memory so that the log_lifecycle
-        # will print out the work unit that is to be associated with the job in the database
-        # via the update_model() call.
-        # We want to log the work_unit_id as early as possible. A failure can happen in between
-        # when we start the job in receptor and when we associate the job <-> work_unit_id.
-        # In that case, there will be work running in receptor and Controller will not know
-        # which Job it is associated with.
-        # We do not programatically handle this case. Ideally, we would handle this with a reaper case.
-        # The two distinct job lifecycle log events below allow for us to at least detect when this
-        # edge case occurs. If the lifecycle event work_unit_id_received occurs without the
-        # work_unit_id_assigned event then this case may have occured.
-        self.task.instance.work_unit_id = result['unitid']  # Set work_unit_id in-memory only
-        self.task.instance.log_lifecycle("work_unit_id_received")
-        self.task.update_model(self.task.instance.pk, work_unit_id=result['unitid'])
-        self.task.instance.log_lifecycle("work_unit_id_assigned")
+            work_submit_kw['node'] = self.task.instance.execution_node
+            use_stream_tls = get_conn_type(work_submit_kw['node'], receptor_ctl).name == "STREAMTLS"
+            work_submit_kw['tlsclient'] = get_tls_client(use_stream_tls)
 
-        sockin.close()
-        sockout.close()
+        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+            transmitter_future = executor.submit(self.transmit, sockin)
 
-        if transmitter_thread.exc:
-            raise transmitter_thread.exc[1].with_traceback(transmitter_thread.exc[2])
+            # submit our work, passing in the right side of our socketpair for reading.
+            result = receptor_ctl.submit_work(payload=sockout.makefile('rb'), **work_submit_kw)
 
-        transmitter_thread.join()
+            sockin.close()
+            sockout.close()
+
+            self.unit_id = result['unitid']
+            # Update the job with the work unit in-memory so that the log_lifecycle
+            # will print out the work unit that is to be associated with the job in the database
+            # via the update_model() call.
+            # We want to log the work_unit_id as early as possible. A failure can happen in between
+            # when we start the job in receptor and when we associate the job <-> work_unit_id.
+            # In that case, there will be work running in receptor and Controller will not know
+            # which Job it is associated with.
+            # We do not programatically handle this case. Ideally, we would handle this with a reaper case.
+            # The two distinct job lifecycle log events below allow for us to at least detect when this
+            # edge case occurs. If the lifecycle event work_unit_id_received occurs without the
+            # work_unit_id_assigned event then this case may have occured.
+            self.task.instance.work_unit_id = result['unitid']  # Set work_unit_id in-memory only
+            self.task.instance.log_lifecycle("work_unit_id_received")
+            self.task.update_model(self.task.instance.pk, work_unit_id=result['unitid'])
+            self.task.instance.log_lifecycle("work_unit_id_assigned")
+
+        # Throws an exception if the transmit failed.
+        # Will be caught by the try/except in BaseTask#run.
+        transmitter_future.result()
 
         # Artifacts are an output, but sometimes they are an input as well
         # this is the case with fact cache, where clearing facts deletes a file, and this must be captured
         artifact_dir = os.path.join(self.runner_params['private_data_dir'], 'artifacts')
-        if os.path.exists(artifact_dir):
+        if self.work_type != 'local' and os.path.exists(artifact_dir):
             shutil.rmtree(artifact_dir)
 
         resultsock, resultfile = receptor_ctl.get_work_results(self.unit_id, return_socket=True, return_sockfile=True)
@@ -367,9 +358,9 @@ class AWXReceptorJob:
                     logger.exception(f'An error was encountered while getting status for work unit {self.unit_id}')
 
                 if 'exceeded quota' in detail:
-                    logger.warn(detail)
+                    logger.warning(detail)
                     log_name = self.task.instance.log_format
-                    logger.warn(f"Could not launch pod for {log_name}. Exceeded quota.")
+                    logger.warning(f"Could not launch pod for {log_name}. Exceeded quota.")
                     self.task.update_model(self.task.instance.pk, status='pending')
                     return
                 # If ansible-runner ran, but an error occured at runtime, the traceback information
@@ -389,7 +380,7 @@ class AWXReceptorJob:
                             self.task.instance.result_traceback = detail
                             self.task.instance.save(update_fields=['result_traceback'])
                         else:
-                            logger.warn(f'No result details or output from {self.task.instance.log_format}, status:\n{state_name}')
+                            logger.warning(f'No result details or output from {self.task.instance.log_format}, status:\n{state_name}')
                     except Exception:
                         raise RuntimeError(detail)
 
@@ -488,6 +479,48 @@ class AWXReceptorJob:
             if self.task.instance.execution_environment.pull:
                 pod_spec['spec']['containers'][0]['imagePullPolicy'] = pull_options[self.task.instance.execution_environment.pull]
 
+        # This allows the user to also expose the isolated path list
+        # to EEs running in k8s/ocp environments, i.e. container groups.
+        # This assumes the node and SA supports hostPath volumes
+        # type is not passed due to backward compatibility,
+        # which means that no checks will be performed before mounting the hostPath volume.
+        if settings.AWX_MOUNT_ISOLATED_PATHS_ON_K8S and settings.AWX_ISOLATION_SHOW_PATHS:
+            spec_volume_mounts = []
+            spec_volumes = []
+
+            for idx, this_path in enumerate(settings.AWX_ISOLATION_SHOW_PATHS):
+                mount_option = None
+                if this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER:
+                    src, dest, mount_option = this_path.split(':')
+                elif this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER - 1:
+                    src, dest = this_path.split(':')
+                else:
+                    src = dest = this_path
+
+                # Enforce read-only volume if 'ro' has been explicitly passed
+                # We do this so we can use the same configuration for regular scenarios and k8s
+                # Since flags like ':O', ':z' or ':Z' are not valid in the k8s realm
+                # Example: /data:/data:ro
+                read_only = bool('ro' == mount_option)
+
+                # Since type is not being passed, k8s by default will not perform any checks if the
+                # hostPath volume exists on the k8s node itself.
+                spec_volumes.append({'name': f'volume-{idx}', 'hostPath': {'path': src}})
+
+                spec_volume_mounts.append({'name': f'volume-{idx}', 'mountPath': f'{dest}', 'readOnly': read_only})
+
+            # merge any volumes definition already present in the pod_spec
+            if 'volumes' in pod_spec['spec']:
+                pod_spec['spec']['volumes'] += spec_volumes
+            else:
+                pod_spec['spec']['volumes'] = spec_volumes
+
+            # merge any volumesMounts definition already present in the pod_spec
+            if 'volumeMounts' in pod_spec['spec']['containers'][0]:
+                pod_spec['spec']['containers'][0]['volumeMounts'] += spec_volume_mounts
+            else:
+                pod_spec['spec']['containers'][0]['volumeMounts'] = spec_volume_mounts
+
         if self.task and self.task.instance.is_container_group_task:
             # If EE credential is passed, create an imagePullSecret
             if self.task.instance.execution_environment and self.task.instance.execution_environment.credential:

awx/main/tasks/system.py

@@ -1,5 +1,6 @@
 # Python
 from collections import namedtuple
+import itertools
 import functools
 import importlib
 import json
@@ -13,15 +14,16 @@ from distutils.version import LooseVersion as Version
 
 # Django
 from django.conf import settings
-from django.db import transaction, DatabaseError, IntegrityError
+from django.db import connection, transaction, DatabaseError, IntegrityError
 from django.db.models.fields.related import ForeignKey
 from django.utils.timezone import now
 from django.utils.encoding import smart_str
 from django.contrib.auth.models import User
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.utils.translation import gettext_noop
 from django.core.cache import cache
 from django.core.exceptions import ObjectDoesNotExist
+from django.contrib.contenttypes.models import ContentType
 
 # Django-CRUM
 from crum import impersonate
@@ -46,6 +48,7 @@ from awx.main.models import (
     Inventory,
     SmartInventoryMembership,
     Job,
+    convert_jsonfields_to_jsonb,
 )
 from awx.main.constants import ACTIVE_STATES
 from awx.main.dispatch.publish import task
@@ -78,6 +81,9 @@ Try upgrading OpenSSH or providing your private key in an different format. \
 
 def dispatch_startup():
     startup_logger = logging.getLogger('awx.main.tasks')
+
+    convert_jsonfields_to_jsonb()
+
     startup_logger.debug("Syncing Schedules")
     for sch in Schedule.objects.all():
         try:
@@ -121,6 +127,123 @@ def inform_cluster_of_shutdown():
         logger.exception('Encountered problem with normal shutdown signal.')
 
 
+def migrate_json_fields_expensive(table, columns):
+    batchsize = 50000
+
+    ct = ContentType.objects.get_by_natural_key(*table.split('_', 1))
+    model = ct.model_class()
+
+    # Phase 1: add the new columns, making them nullable to avoid populating them
+    with connection.schema_editor() as schema_editor:
+        # See: https://docs.djangoproject.com/en/3.1/ref/schema-editor/
+
+        for colname in columns:
+            f = model._meta.get_field(colname)
+            _, _, args, kwargs = f.deconstruct()
+            kwargs['null'] = True
+            new_f = f.__class__(*args, **kwargs)
+            new_f.set_attributes_from_name(f'_{colname}')
+
+            schema_editor.add_field(model, new_f)
+
+            # Create a trigger to make sure new data automatically gets put in both fields.
+            with connection.cursor() as cursor:
+                # It's a little annoying, I think this trigger will re-do
+                # the same work as the update query in Phase 2
+                cursor.execute(
+                    f"""
+                    create or replace function update_{table}_{colname}()
+                      returns trigger as $body$
+                      begin
+                        new._{colname} = new.{colname}::jsonb
+                        return new;
+                      end
+                      $body$ language plpgsql;
+                    """
+                )
+                cursor.execute(
+                    f"""
+                    create trigger {table}_{colname}_trigger
+                      before insert or update
+                      on {table}
+                      for each row
+                      execute procedure update_{table}_{colname};
+                    """
+                )
+
+    # Phase 2: copy over the data
+    with connection.cursor() as cursor:
+        rows = 0
+        for i in itertools.count(0, batchsize):
+            cursor.execute(f"select count(1) from {table} where id >= %s;", (i,))
+            if not cursor.fetchone()[0]:
+                break
+
+            column_expr = ', '.join(f"_{colname} = {colname}::jsonb" for colname in columns)
+            cursor.execute(
+                f"""
+                update {table}
+                  set {column_expr}
+                  where id >= %s and id < %s;
+                """,
+                (i, i + batchsize),
+            )
+            rows += cursor.rowcount
+            logger.debug(f"Batch {i} to {i + batchsize} copied on {table}.")
+
+        logger.warning(f"Data copied for {rows} rows on {table}.")
+
+    # Phase 3: drop the old column and rename the new one
+    with connection.schema_editor() as schema_editor:
+
+        # FIXME: Grab a lock explicitly here?
+        for colname in columns:
+            with connection.cursor() as cursor:
+                cursor.execute(f"drop trigger {table}_{colname}_trigger;")
+                cursor.execute(f"drop function update_{table}_{colname};")
+
+            f = model._meta.get_field(colname)
+            _, _, args, kwargs = f.deconstruct()
+            kwargs['null'] = True
+            new_f = f.__class__(*args, **kwargs)
+            new_f.set_attributes_from_name(f'_{colname}')
+
+            schema_editor.remove_field(model, f)
+
+            _, _, args, kwargs = new_f.deconstruct()
+            f = new_f.__class__(*args, **kwargs)
+            f.set_attributes_from_name(colname)
+
+            schema_editor.alter_field(model, new_f, f)
+
+
+@task(queue=get_local_queuename)
+def migrate_json_fields(table, expensive, columns):
+    logger.warning(f"Migrating json fields: {table} {columns}")
+
+    with advisory_lock(f'json_migration_{table}', wait=False) as acquired:
+        if not acquired:
+            return
+
+        from django.db.migrations.executor import MigrationExecutor
+
+        # If Django is currently running migrations, wait until it is done.
+        while True:
+            executor = MigrationExecutor(connection)
+            if not executor.migration_plan(executor.loader.graph.leaf_nodes()):
+                break
+            time.sleep(60)
+
+        if expensive:
+            migrate_json_fields_expensive(table, columns)
+        else:
+            with connection.cursor() as cursor:
+                column_expr = " ".join(f"ALTER {colname} TYPE jsonb" for colname in columns)
+                cursor.execute(f"ALTER TABLE {table} {column_expr};")
+
+    logger.warning(f"Migration of {table} to jsonb is finished")
+
+
 @task(queue=get_local_queuename)
 def apply_cluster_membership_policies():
     from awx.main.signals import disable_activity_stream
@@ -374,15 +497,15 @@ def cluster_node_health_check(node):
     Used for the health check endpoint, refreshes the status of the instance, but must be ran on target node
     """
     if node == '':
-        logger.warn('Local health check incorrectly called with blank string')
+        logger.warning('Local health check incorrectly called with blank string')
         return
     elif node != settings.CLUSTER_HOST_ID:
-        logger.warn(f'Local health check for {node} incorrectly sent to {settings.CLUSTER_HOST_ID}')
+        logger.warning(f'Local health check for {node} incorrectly sent to {settings.CLUSTER_HOST_ID}')
         return
     try:
         this_inst = Instance.objects.me()
     except Instance.DoesNotExist:
-        logger.warn(f'Instance record for {node} missing, could not check capacity.')
+        logger.warning(f'Instance record for {node} missing, could not check capacity.')
         return
     this_inst.local_health_check()
 
@@ -390,12 +513,12 @@ def cluster_node_health_check(node):
 @task(queue=get_local_queuename)
 def execution_node_health_check(node):
     if node == '':
-        logger.warn('Remote health check incorrectly called with blank string')
+        logger.warning('Remote health check incorrectly called with blank string')
         return
     try:
         instance = Instance.objects.get(hostname=node)
     except Instance.DoesNotExist:
-        logger.warn(f'Instance record for {node} missing, could not check capacity.')
+        logger.warning(f'Instance record for {node} missing, could not check capacity.')
         return
 
     if instance.node_type != 'execution':
@@ -416,7 +539,7 @@ def execution_node_health_check(node):
     if data['errors']:
         formatted_error = "\n".join(data["errors"])
         if prior_capacity:
-            logger.warn(f'Health check marking execution node {node} as lost, errors:\n{formatted_error}')
+            logger.warning(f'Health check marking execution node {node} as lost, errors:\n{formatted_error}')
         else:
             logger.info(f'Failed to find capacity of new or lost execution node {node}, errors:\n{formatted_error}')
     else:
@@ -436,12 +559,11 @@ def inspect_execution_nodes(instance_list):
         workers = mesh_status['Advertisements']
         for ad in workers:
             hostname = ad['NodeID']
-            changed = False
 
             if hostname in node_lookup:
                 instance = node_lookup[hostname]
             else:
-                logger.warn(f"Unrecognized node advertising on mesh: {hostname}")
+                logger.warning(f"Unrecognized node advertising on mesh: {hostname}")
                 continue
 
             # Control-plane nodes are dealt with via local_health_check instead.
@@ -458,15 +580,16 @@ def inspect_execution_nodes(instance_list):
 
             # Only execution nodes should be dealt with by execution_node_health_check
             if instance.node_type == 'hop':
+                if was_lost and (not instance.is_lost(ref_time=nowtime)):
+                    logger.warning(f'Hop node {hostname}, has rejoined the receptor mesh')
+                    instance.save_health_data(errors='')
                 continue
 
-            if changed:
-                execution_node_health_check.apply_async([hostname])
-            elif was_lost:
+            if was_lost:
                 # if the instance *was* lost, but has appeared again,
                 # attempt to re-establish the initial capacity and version
                 # check
-                logger.warn(f'Execution node attempting to rejoin as instance {hostname}.')
+                logger.warning(f'Execution node attempting to rejoin as instance {hostname}.')
                 execution_node_health_check.apply_async([hostname])
             elif instance.capacity == 0 and instance.enabled:
                 # nodes with proven connection but need remediation run health checks are reduced frequency
@@ -534,20 +657,14 @@ def cluster_node_heartbeat():
         except Exception:
             logger.exception('failed to reap jobs for {}'.format(other_inst.hostname))
         try:
-            # Capacity could already be 0 because:
-            #  * It's a new node and it never had a heartbeat
-            #  * It was set to 0 by another tower node running this method
-            #  * It was set to 0 by this node, but auto deprovisioning is off
-            #
-            # If auto deprovisioning is on, don't bother setting the capacity to 0
-            # since we will delete the node anyway.
-            if other_inst.capacity != 0 and not settings.AWX_AUTO_DEPROVISION_INSTANCES:
-                other_inst.mark_offline(errors=_('Another cluster node has determined this instance to be unresponsive'))
-                logger.error("Host {} last checked in at {}, marked as lost.".format(other_inst.hostname, other_inst.last_seen))
-            elif settings.AWX_AUTO_DEPROVISION_INSTANCES:
+            if settings.AWX_AUTO_DEPROVISION_INSTANCES:
                 deprovision_hostname = other_inst.hostname
                 other_inst.delete()
                 logger.info("Host {} Automatically Deprovisioned.".format(deprovision_hostname))
+            elif other_inst.capacity != 0 or (not other_inst.errors):
+                other_inst.mark_offline(errors=_('Another cluster node has determined this instance to be unresponsive'))
+                logger.error("Host {} last checked in at {}, marked as lost.".format(other_inst.hostname, other_inst.last_seen))
+
         except DatabaseError as e:
             if 'did not affect any rows' in str(e):
                 logger.debug('Another instance has marked {} as lost'.format(other_inst.hostname))
@@ -640,7 +757,7 @@ def awx_periodic_scheduler():
             template = schedule.unified_job_template
             schedule.update_computed_fields()  # To update next_run timestamp.
             if template.cache_timeout_blocked:
-                logger.warn("Cache timeout is in the future, bypassing schedule for template %s" % str(template.id))
+                logger.warning("Cache timeout is in the future, bypassing schedule for template %s" % str(template.id))
                 continue
             try:
                 job_kwargs = schedule.get_job_kwargs()
@@ -694,7 +811,7 @@ def handle_work_error(task_id, *args, **kwargs):
                 instance = UnifiedJob.get_instance_by_type(each_task['type'], each_task['id'])
                 if not instance:
                     # Unknown task type
-                    logger.warn("Unknown task type: {}".format(each_task['type']))
+                    logger.warning("Unknown task type: {}".format(each_task['type']))
                     continue
             except ObjectDoesNotExist:
                 logger.warning('Missing {} `{}` in error callback.'.format(each_task['type'], each_task['id']))
@@ -741,7 +858,7 @@ def handle_success_and_failure_notifications(job_id):
             time.sleep(1)
             uj = UnifiedJob.objects.get(pk=job_id)
 
-    logger.warn(f"Failed to even try to send notifications for job '{uj}' due to job not being in finished state.")
+    logger.warning(f"Failed to even try to send notifications for job '{uj}' due to job not being in finished state.")
 
 
 @task(queue=get_local_queuename)

awx/main/tests/conftest.py

@@ -3,7 +3,7 @@ import pytest
 from unittest import mock
 from contextlib import contextmanager
 
-from awx.main.models import Credential, UnifiedJob
+from awx.main.models import Credential, UnifiedJob, Instance
 from awx.main.tests.factories import (
     create_organization,
     create_job_template,
@@ -212,3 +212,10 @@ def mock_get_event_queryset_no_job_created():
 
     with mock.patch.object(UnifiedJob, 'get_event_queryset', lambda self: event_qs(self)) as _fixture:
         yield _fixture
+
+
+@pytest.fixture
+def mock_me():
+    me_mock = mock.MagicMock(return_value=Instance(id=1, hostname=settings.CLUSTER_HOST_ID, uuid='00000000-0000-0000-0000-000000000000'))
+    with mock.patch.object(Instance.objects, 'me', me_mock):
+        yield

awx/main/tests/docs/test_swagger_generation.py

@@ -5,7 +5,7 @@ import re
 from django.conf import settings
 from django.core.serializers.json import DjangoJSONEncoder
 from django.utils.functional import Promise
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 
 from openapi_codec.encode import generate_swagger_object
 import pytest
@@ -16,9 +16,9 @@ from awx.api.versioning import drf_reverse
 class i18nEncoder(DjangoJSONEncoder):
     def default(self, obj):
         if isinstance(obj, Promise):
-            return force_text(obj)
+            return force_str(obj)
         if type(obj) == bytes:
-            return force_text(obj)
+            return force_str(obj)
         return super(i18nEncoder, self).default(obj)
 
 

awx/main/tests/factories/fixtures.py

@@ -180,8 +180,8 @@ def mk_job_template(
 
     jt.project = project
 
-    jt.survey_spec = spec
-    if jt.survey_spec is not None:
+    if spec is not None:
+        jt.survey_spec = spec
         jt.survey_enabled = True
 
     if persisted:
@@ -212,8 +212,8 @@ def mk_workflow_job_template(name, extra_vars='', spec=None, organization=None,
 
     wfjt = WorkflowJobTemplate(name=name, extra_vars=extra_vars, organization=organization, webhook_service=webhook_service)
 
-    wfjt.survey_spec = spec
-    if wfjt.survey_spec:
+    if spec:
+        wfjt.survey_spec = spec
         wfjt.survey_enabled = True
 
     if persisted:

awx/main/tests/functional/api/test_unified_jobs_stdout.py

@@ -3,11 +3,12 @@
 import base64
 import json
 import re
-from datetime import datetime
+from unittest import mock
 
 from django.conf import settings
 from django.utils.encoding import smart_str
-from unittest import mock
+from django.utils.timezone import now as tz_now
+
 import pytest
 
 from awx.api.versioning import reverse
@@ -146,7 +147,7 @@ def test_stdout_line_range(sqlite_copy_expert, Parent, Child, relation, view, ge
 
 @pytest.mark.django_db
 def test_text_stdout_from_system_job_events(sqlite_copy_expert, get, admin):
-    created = datetime.utcnow()
+    created = tz_now()
     job = SystemJob(created=created)
     job.save()
     for i in range(3):
@@ -158,7 +159,7 @@ def test_text_stdout_from_system_job_events(sqlite_copy_expert, get, admin):
 
 @pytest.mark.django_db
 def test_text_stdout_with_max_stdout(sqlite_copy_expert, get, admin):
-    created = datetime.utcnow()
+    created = tz_now()
     job = SystemJob(created=created)
     job.save()
     total_bytes = settings.STDOUT_MAX_BYTES_DISPLAY + 1
@@ -185,7 +186,7 @@ def test_text_stdout_with_max_stdout(sqlite_copy_expert, get, admin):
 @pytest.mark.parametrize('fmt', ['txt', 'ansi'])
 @mock.patch('awx.main.redact.UriCleaner.SENSITIVE_URI_PATTERN', mock.Mock(**{'search.return_value': None}))  # really slow for large strings
 def test_max_bytes_display(sqlite_copy_expert, Parent, Child, relation, view, fmt, get, admin):
-    created = datetime.utcnow()
+    created = tz_now()
     job = Parent(created=created)
     job.save()
     total_bytes = settings.STDOUT_MAX_BYTES_DISPLAY + 1
@@ -267,7 +268,7 @@ def test_text_with_unicode_stdout(sqlite_copy_expert, Parent, Child, relation, v
 
 @pytest.mark.django_db
 def test_unicode_with_base64_ansi(sqlite_copy_expert, get, admin):
-    created = datetime.utcnow()
+    created = tz_now()
     job = Job(created=created)
     job.save()
     for i in range(3):

awx/main/tests/functional/api/test_user.py

@@ -1,4 +1,5 @@
 from datetime import date
+from unittest import mock
 
 import pytest
 
@@ -17,7 +18,7 @@ EXAMPLE_USER_DATA = {"username": "affable", "first_name": "a", "last_name": "a",
 
 @pytest.mark.django_db
 def test_user_create(post, admin):
-    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware())
+    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware(mock.Mock()))
     assert response.status_code == 201
     assert not response.data['is_superuser']
     assert not response.data['is_system_auditor']
@@ -25,22 +26,22 @@ def test_user_create(post, admin):
 
 @pytest.mark.django_db
 def test_fail_double_create_user(post, admin):
-    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware())
+    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware(mock.Mock()))
     assert response.status_code == 201
 
-    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware())
+    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware(mock.Mock()))
     assert response.status_code == 400
 
 
 @pytest.mark.django_db
 def test_create_delete_create_user(post, delete, admin):
-    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware())
+    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware(mock.Mock()))
     assert response.status_code == 201
 
-    response = delete(reverse('api:user_detail', kwargs={'pk': response.data['id']}), admin, middleware=SessionMiddleware())
+    response = delete(reverse('api:user_detail', kwargs={'pk': response.data['id']}), admin, middleware=SessionMiddleware(mock.Mock()))
     assert response.status_code == 204
 
-    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware())
+    response = post(reverse('api:user_list'), EXAMPLE_USER_DATA, admin, middleware=SessionMiddleware(mock.Mock()))
     print(response.data)
     assert response.status_code == 201
 
@@ -48,7 +49,7 @@ def test_create_delete_create_user(post, delete, admin):
 @pytest.mark.django_db
 def test_user_cannot_update_last_login(patch, admin):
     assert admin.last_login is None
-    patch(reverse('api:user_detail', kwargs={'pk': admin.pk}), {'last_login': '2020-03-13T16:39:47.303016Z'}, admin, middleware=SessionMiddleware())
+    patch(reverse('api:user_detail', kwargs={'pk': admin.pk}), {'last_login': '2020-03-13T16:39:47.303016Z'}, admin, middleware=SessionMiddleware(mock.Mock()))
     assert User.objects.get(pk=admin.pk).last_login is None
 
 

awx/main/tests/functional/commands/test_cleanup_jobs.py

@@ -1,178 +0,0 @@
-import pytest
-from datetime import datetime, timedelta
-from pytz import timezone
-from collections import OrderedDict
-from unittest import mock
-
-from django.db.models.deletion import Collector, SET_NULL, CASCADE
-from django.core.management import call_command
-
-from awx.main.management.commands import cleanup_jobs
-from awx.main.utils.deletion import AWXCollector
-from awx.main.models import JobTemplate, User, Job, Notification, WorkflowJobNode, JobHostSummary
-
-
-@pytest.fixture
-def setup_environment(inventory, project, machine_credential, host, notification_template, label):
-    """
-    Create old jobs and new jobs, with various other objects to hit the
-    related fields of Jobs. This makes sure on_delete() effects are tested
-    properly.
-    """
-    old_jobs = []
-    new_jobs = []
-    days = 10
-    days_str = str(days)
-
-    jt = JobTemplate.objects.create(name='testjt', inventory=inventory, project=project)
-    jt.credentials.add(machine_credential)
-    jt_user = User.objects.create(username='jobtemplateuser')
-    jt.execute_role.members.add(jt_user)
-
-    notification = Notification()
-    notification.notification_template = notification_template
-    notification.save()
-
-    for i in range(3):
-        # create jobs with current time
-        job1 = jt.create_job()
-        job1.created = datetime.now(tz=timezone('UTC'))
-        job1.save()
-        # sqlite does not support partitioning so we cannot test partition-based jobevent cleanup
-        # JobEvent.create_from_data(job_id=job1.pk, uuid='abc123', event='runner_on_start', stdout='a' * 1025).save()
-        new_jobs.append(job1)
-
-        # create jobs 10 days ago
-        job2 = jt.create_job()
-        job2.created = datetime.now(tz=timezone('UTC')) - timedelta(days=days)
-        job2.save()
-        job2.dependent_jobs.add(job1)
-        # JobEvent.create_from_data(job_id=job2.pk, uuid='abc123', event='runner_on_start', stdout='a' * 1025).save()
-        old_jobs.append(job2)
-
-    jt.last_job = job2
-    jt.current_job = job2
-    jt.save()
-    host.last_job = job2
-    host.save()
-    notification.unifiedjob_notifications.add(job2)
-    label.unifiedjob_labels.add(job2)
-    jn = WorkflowJobNode.objects.create(job=job2)
-    jn.save()
-    jh = JobHostSummary.objects.create(job=job2)
-    jh.save()
-
-    return (old_jobs, new_jobs, days_str)
-
-
-# sqlite does not support table partitioning so we mock out the methods responsible for pruning
-# job event partitions during the job cleanup task
-# https://github.com/ansible/awx/issues/9039
-@pytest.mark.django_db
-@mock.patch.object(cleanup_jobs.DeleteMeta, 'identify_excluded_partitions', mock.MagicMock())
-@mock.patch.object(cleanup_jobs.DeleteMeta, 'find_partitions_to_drop', mock.MagicMock())
-@mock.patch.object(cleanup_jobs.DeleteMeta, 'drop_partitions', mock.MagicMock())
-def test_cleanup_jobs(setup_environment):
-    (old_jobs, new_jobs, days_str) = setup_environment
-
-    # related_fields
-    related = [f for f in Job._meta.get_fields(include_hidden=True) if f.auto_created and not f.concrete and (f.one_to_one or f.one_to_many)]
-
-    job = old_jobs[-1]  # last job
-
-    # gather related objects for job
-    related_should_be_removed = {}
-    related_should_be_null = {}
-    for r in related:
-        qs = r.related_model._base_manager.using('default').filter(**{"%s__in" % r.field.name: [job.pk]})
-        if qs.exists():
-            if r.field.remote_field.on_delete == CASCADE:
-                related_should_be_removed[qs.model] = set(qs.values_list('pk', flat=True))
-            if r.field.remote_field.on_delete == SET_NULL:
-                related_should_be_null[(qs.model, r.field.name)] = set(qs.values_list('pk', flat=True))
-
-    assert related_should_be_removed
-    assert related_should_be_null
-
-    call_command('cleanup_jobs', '--days', days_str)
-    # make sure old jobs are removed
-    assert not Job.objects.filter(pk__in=[obj.pk for obj in old_jobs]).exists()
-
-    # make sure new jobs are untouched
-    assert len(new_jobs) == Job.objects.filter(pk__in=[obj.pk for obj in new_jobs]).count()
-
-    # make sure related objects are destroyed or set to NULL (none)
-    for model, values in related_should_be_removed.items():
-        assert not model.objects.filter(pk__in=values).exists()
-
-    for (model, fieldname), values in related_should_be_null.items():
-        for v in values:
-            assert not getattr(model.objects.get(pk=v), fieldname)
-
-
-@pytest.mark.django_db
-def test_awxcollector(setup_environment):
-    """
-    Efforts to improve the performance of cleanup_jobs involved
-    sub-classing the django Collector class. This unit test will
-    check for parity between the django Collector and the modified
-    AWXCollector class. AWXCollector is used in cleanup_jobs to
-    bulk-delete old jobs from the database.
-
-    Specifically, Collector has four dictionaries to check:
-    .dependencies, .data, .fast_deletes, and .field_updates
-
-    These tests will convert each dictionary from AWXCollector
-    (after running .collect on jobs), from querysets to sets of
-    objects. The final result should be a dictionary that is
-    equivalent to django's Collector.
-    """
-
-    (old_jobs, new_jobs, days_str) = setup_environment
-    collector = Collector('default')
-    collector.collect(old_jobs)
-
-    awx_col = AWXCollector('default')
-    # awx_col accepts a queryset as input
-    awx_col.collect(Job.objects.filter(pk__in=[obj.pk for obj in old_jobs]))
-
-    # check that dependencies are the same
-    assert awx_col.dependencies == collector.dependencies
-
-    # check that objects to delete are the same
-    awx_del_dict = OrderedDict()
-    for model, instances in awx_col.data.items():
-        awx_del_dict.setdefault(model, set())
-        for inst in instances:
-            # .update() will put each object in a queryset into the set
-            awx_del_dict[model].update(inst)
-    assert awx_del_dict == collector.data
-
-    # check that field updates are the same
-    awx_del_dict = OrderedDict()
-    for model, instances_for_fieldvalues in awx_col.field_updates.items():
-        awx_del_dict.setdefault(model, {})
-        for (field, value), instances in instances_for_fieldvalues.items():
-            awx_del_dict[model].setdefault((field, value), set())
-            for inst in instances:
-                awx_del_dict[model][(field, value)].update(inst)
-
-    # collector field updates don't use the base (polymorphic parent) model, e.g.
-    # it will use JobTemplate instead of UnifiedJobTemplate. Therefore,
-    # we need to rebuild the dictionary and grab the model from the field
-    collector_del_dict = OrderedDict()
-    for model, instances_for_fieldvalues in collector.field_updates.items():
-        for (field, value), instances in instances_for_fieldvalues.items():
-            collector_del_dict.setdefault(field.model, {})
-            collector_del_dict[field.model][(field, value)] = collector.field_updates[model][(field, value)]
-    assert awx_del_dict == collector_del_dict
-
-    # check that fast deletes are the same
-    collector_fast_deletes = set()
-    for q in collector.fast_deletes:
-        collector_fast_deletes.update(q)
-
-    awx_col_fast_deletes = set()
-    for q in awx_col.fast_deletes:
-        awx_col_fast_deletes.update(q)
-    assert collector_fast_deletes == awx_col_fast_deletes

awx/main/tests/functional/conftest.py

@@ -15,7 +15,6 @@ from django.core.serializers.json import DjangoJSONEncoder
 from django.db.backends.sqlite3.base import SQLiteCursorWrapper
 
 # AWX
-from awx.main.fields import JSONBField
 from awx.main.models.projects import Project
 from awx.main.models.ha import Instance
 
@@ -755,11 +754,6 @@ def get_db_prep_save(self, value, connection, **kwargs):
     return value
 
 
-@pytest.fixture
-def monkeypatch_jsonbfield_get_db_prep_save(mocker):
-    JSONBField.get_db_prep_save = get_db_prep_save
-
-
 @pytest.fixture
 def oauth_application(admin):
     return Application.objects.create(name='test app', user=admin, client_type='confidential', authorization_grant_type='password')

awx/main/tests/functional/models/test_inventory.py

@@ -110,6 +110,16 @@ class TestActiveCount:
         source.hosts.create(name='remotely-managed-host', inventory=inventory)
         assert Host.objects.active_count() == 1
 
+    def test_host_case_insensitivity(self, organization):
+        inv1 = Inventory.objects.create(name='inv1', organization=organization)
+        inv2 = Inventory.objects.create(name='inv2', organization=organization)
+        assert Host.objects.active_count() == 0
+        inv1.hosts.create(name='host1')
+        inv2.hosts.create(name='Host1')
+        assert Host.objects.active_count() == 1
+        inv1.hosts.create(name='host2')
+        assert Host.objects.active_count() == 2
+
 
 @pytest.mark.django_db
 class TestSCMUpdateFeatures:

awx/main/tests/functional/test_instances.py

@@ -363,6 +363,23 @@ def test_health_check_oh_no():
     assert instance.errors == 'This it not a real instance!'
 
 
+@pytest.mark.django_db
+def test_errors_field_alone():
+    instance = Instance.objects.create(hostname='foo-1', enabled=True, node_type='hop')
+
+    instance.save_health_data(errors='Node went missing!')
+    assert instance.errors == 'Node went missing!'
+    assert instance.capacity == 0
+    assert instance.memory == instance.mem_capacity == 0
+    assert instance.cpu == instance.cpu_capacity == 0
+
+    instance.save_health_data(errors='')
+    assert not instance.errors
+    assert instance.capacity == 0
+    assert instance.memory == instance.mem_capacity == 0
+    assert instance.cpu == instance.cpu_capacity == 0
+
+
 @pytest.mark.django_db
 class TestInstanceGroupOrdering:
     def test_ad_hoc_instance_groups(self, instance_group_factory, inventory, default_instance_group):

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -181,7 +181,7 @@ def create_reference_data(source_dir, env, content):
 
 @pytest.mark.django_db
 @pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS)
-def test_inventory_update_injected_content(this_kind, inventory, fake_credential_factory):
+def test_inventory_update_injected_content(this_kind, inventory, fake_credential_factory, mock_me):
     ExecutionEnvironment.objects.create(name='Control Plane EE', managed=True)
     ExecutionEnvironment.objects.create(name='Default Job EE', managed=False)
 

awx/main/tests/functional/test_named_url.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+from unittest import mock
+
 import pytest
 
 from django.core.exceptions import ImproperlyConfigured
@@ -31,7 +33,7 @@ def setup_module(module):
     # in unit test environment. So it is wrapped by try-except block to mute any
     # unwanted exceptions.
     try:
-        URLModificationMiddleware()
+        URLModificationMiddleware(mock.Mock())
     except ImproperlyConfigured:
         pass
 

awx/main/tests/functional/test_session.py

@@ -1,16 +1,12 @@
 from importlib import import_module
 import pytest
-import re
 
 from django.conf import settings
 from django.test.utils import override_settings
-from django.contrib.sessions.middleware import SessionMiddleware
 from django.contrib.sessions.models import Session
 from django.contrib.auth import SESSION_KEY
 from unittest import mock
 
-from awx.api.versioning import reverse
-
 
 class AlwaysPassBackend(object):
 
@@ -30,26 +26,6 @@ def test_login_json_not_allowed(get, accept, status):
     get('/api/login/', HTTP_ACCEPT=accept, expect=status)
 
 
-@pytest.mark.skip(reason="Needs Update - CA")
-@pytest.mark.django_db
-def test_session_create_delete(admin, post, get):
-    AlwaysPassBackend.user = admin
-    with override_settings(AUTHENTICATION_BACKENDS=(AlwaysPassBackend.get_backend_path(),), SESSION_COOKIE_NAME='session_id'):
-        response = post(
-            '/api/login/',
-            data={'username': admin.username, 'password': admin.password, 'next': '/api/'},
-            expect=302,
-            middleware=SessionMiddleware(),
-            format='multipart',
-        )
-        assert 'session_id' in response.cookies
-        session_key = re.findall(r'session_id=[a-zA-z0-9]+', str(response.cookies['session_id']))[0][len('session_id=') :]
-        session = Session.objects.get(session_key=session_key)
-        assert int(session.get_decoded()[SESSION_KEY]) == admin.pk
-        response = get('/api/logout/', middleware=SessionMiddleware(), cookies={'session_id': session_key}, expect=302)
-        assert not Session.objects.filter(session_key=session_key).exists()
-
-
 @pytest.mark.django_db
 @mock.patch('awx.main.consumers.emit_channel_notification')
 def test_sessions_unlimited(emit, admin):
@@ -81,21 +57,3 @@ def test_session_overlimit(emit, admin, alice):
     store = import_module(settings.SESSION_ENGINE).SessionStore()
     store.create_model_instance({SESSION_KEY: alice.pk}).save()
     assert Session.objects.count() == 4
-
-
-@pytest.mark.skip(reason="Needs Update - CA")
-@pytest.mark.django_db
-def test_password_update_clears_sessions(admin, alice, post, patch):
-    AlwaysPassBackend.user = alice
-    with override_settings(AUTHENTICATION_BACKENDS=(AlwaysPassBackend.get_backend_path(),), SESSION_COOKIE_NAME='session_id'):
-        response = post(
-            '/api/login/',
-            data={'username': alice.username, 'password': alice.password, 'next': '/api/'},
-            expect=302,
-            middleware=SessionMiddleware(),
-            format='multipart',
-        )
-        session_key = re.findall(r'session_id=[a-zA-z0-9]+', str(response.cookies['session_id']))[0][len('session_id=') :]
-        assert Session.objects.filter(session_key=session_key).exists()
-        patch(reverse('api:user_detail', kwargs={'pk': alice.pk}), admin, data={'password': 'new_password'}, expect=200)
-        assert not Session.objects.filter(session_key=session_key).exists()

awx/main/tests/functional/test_tasks.py

@@ -27,7 +27,7 @@ def test_no_worker_info_on_AWX_nodes(node_type):
 
 @pytest.mark.django_db
 class TestDependentInventoryUpdate:
-    def test_dependent_inventory_updates_is_called(self, scm_inventory_source, scm_revision_file):
+    def test_dependent_inventory_updates_is_called(self, scm_inventory_source, scm_revision_file, mock_me):
         task = RunProjectUpdate()
         task.revision_path = scm_revision_file
         proj_update = scm_inventory_source.source_project.create_project_update()
@@ -36,7 +36,7 @@ class TestDependentInventoryUpdate:
                 task.post_run_hook(proj_update, 'successful')
                 inv_update_mck.assert_called_once_with(proj_update, mock.ANY)
 
-    def test_no_unwanted_dependent_inventory_updates(self, project, scm_revision_file):
+    def test_no_unwanted_dependent_inventory_updates(self, project, scm_revision_file, mock_me):
         task = RunProjectUpdate()
         task.revision_path = scm_revision_file
         proj_update = project.create_project_update()
@@ -45,7 +45,7 @@ class TestDependentInventoryUpdate:
                 task.post_run_hook(proj_update, 'successful')
                 assert not inv_update_mck.called
 
-    def test_dependent_inventory_updates(self, scm_inventory_source, default_instance_group):
+    def test_dependent_inventory_updates(self, scm_inventory_source, default_instance_group, mock_me):
         task = RunProjectUpdate()
         scm_inventory_source.scm_last_revision = ''
         proj_update = ProjectUpdate.objects.create(project=scm_inventory_source.source_project)
@@ -57,7 +57,7 @@ class TestDependentInventoryUpdate:
                 iu_run_mock.assert_called_once_with(inv_update.id)
                 assert inv_update.source_project_update_id == proj_update.pk
 
-    def test_dependent_inventory_project_cancel(self, project, inventory, default_instance_group):
+    def test_dependent_inventory_project_cancel(self, project, inventory, default_instance_group, mock_me):
         """
         Test that dependent inventory updates exhibit good behavior on cancel
         of the source project update

awx/main/tests/unit/api/test_filters.py

@@ -2,7 +2,11 @@
 
 import pytest
 
+# Django
+from django.core.exceptions import FieldDoesNotExist
+
 from rest_framework.exceptions import PermissionDenied, ParseError
+
 from awx.api.filters import FieldLookupBackend, OrderByBackend, get_field_from_path
 from awx.main.models import (
     AdHocCommand,
@@ -22,9 +26,6 @@ from awx.main.models import (
 from awx.main.models.oauth import OAuth2Application
 from awx.main.models.jobs import JobOptions
 
-# Django
-from django.db.models.fields import FieldDoesNotExist
-
 
 def test_related():
     field_lookup = FieldLookupBackend()

awx/main/tests/unit/models/test_credential.py

@@ -1,12 +1,15 @@
 # -*- coding: utf-8 -*-
 
+import pytest
+
 from awx.main.models import Credential, CredentialType
 
 
+@pytest.mark.django_db
 def test_unique_hash_with_unicode():
-    ct = CredentialType(name=u'Väult', kind='vault')
-    cred = Credential(id=4, name=u'Iñtërnâtiônàlizætiøn', credential_type=ct, inputs={u'vault_id': u'🐉🐉🐉'}, credential_type_id=42)
-    assert cred.unique_hash(display=True) == u'Väult (id=🐉🐉🐉)'
+    ct = CredentialType.objects.create(name='Väult', kind='vault')
+    cred = Credential.objects.create(name='Iñtërnâtiônàlizætiøn', credential_type=ct, inputs={'vault_id': '🐉🐉🐉'})
+    assert cred.unique_hash(display=True) == 'Väult (id=🐉🐉🐉)'
 
 
 def test_custom_cred_with_empty_encrypted_field():

awx/main/tests/unit/models/test_survey_models.py

@@ -59,6 +59,38 @@ class SurveyVariableValidation:
         assert accepted == {}
         assert str(errors[0]) == "Value 5 for 'a' expected to be a string."
 
+    def test_job_template_survey_default_variable_validation(self, job_template_factory):
+        objects = job_template_factory(
+            "survey_variable_validation",
+            organization="org1",
+            inventory="inventory1",
+            credential="cred1",
+            persisted=False,
+        )
+        obj = objects.job_template
+        obj.survey_spec = {
+            "description": "",
+            "spec": [
+                {
+                    "required": True,
+                    "min": 0,
+                    "default": "2",
+                    "max": 1024,
+                    "question_description": "",
+                    "choices": "",
+                    "variable": "a",
+                    "question_name": "float_number",
+                    "type": "float",
+                }
+            ],
+            "name": "",
+        }
+
+        obj.survey_enabled = True
+        accepted, _, errors = obj.accept_or_ignore_variables({"a": 2})
+        assert accepted == {{"a": 2.0}}
+        assert not errors
+
 
 @pytest.fixture
 def job(mocker):