Upgrade to Django 3.0

- upgrades
  - Django 3.0.14
  - django-jsonfield 1.4.1 (from 1.2.0)
  - django-oauth-toolkit 1.4.1 (from 1.1.3)
    - Stopping here because later versions have changes to the
      underlying model to support OpenID Connect.  Presumably this can
      be dealt with via a migration in our project.
  - django-guid 2.2.1 (from 2.2.0)
  - django-debug-toolbar 3.2.4 (from 1.11.1)
  - python3-saml 1.13.0 (from 1.9.0)
  - xmlsec 1.3.12 (from 1.3.3)

- Remove our project's use of django.utils.six in favor of directly
  using six, in awx.sso.fields.

- Temporarily monkey patch six back in as django.utils.six, since
  django-jsonfield makes use of that import, and is no longer being
  updated.  Hopefully we can do away with this dependency with the new
  generalized JSONField brought in with Django 3.1.

- Force a json decoder to be used with all instances of JSONField
  brought in by django-jsonfield.  This deals with the 'cast to text'
  problem noted previously in our UPGRADE_BLOCKERS.

- Remove the validate_uris validator from the OAuth2Application in
  migration 0025, per the UPGRADE_BLOCKERS, and remove that note.

- Update the TEMPLATES setting to satisfy Django Debug Toolbar.  It
  requires at least one entry that has APP_DIRS=True, and as near as I
  can tell our custom OPTIONS.loaders setting was effectively doing
  the same thing as Django's own machinery if this setting is set.

awx/__init__.py

@@ -6,6 +6,8 @@ import os
 import sys
 import warnings
 
+import six
+
 from pkg_resources import get_distribution
 
 __version__ = get_distribution('awx').version
@@ -35,7 +37,9 @@ else:
     from django.db.models import indexes
     from django.db.backends.utils import names_digest
     from django.db import connection
+    from django import utils
 
+    utils.six = six  # FIXME: monkey patch to get us through for now
 
 if HAS_DJANGO is True:
 

awx/main/fields.py

@@ -72,6 +72,10 @@ Draft4Validator.VALIDATORS['enum'] = __enum_validate__
 
 
 class JSONField(upstream_JSONField):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.decoder_kwargs = {'cls': json.JSONDecoder}  # FIXME
+
     def db_type(self, connection):
         return 'text'
 

awx/main/migrations/0025_v330_add_oauth_activity_stream_registrar.py

@@ -29,7 +29,7 @@ class Migration(migrations.Migration):
                 ('client_id', models.CharField(db_index=True, default=oauth2_provider.generators.generate_client_id, max_length=100, unique=True)),
                 (
                     'redirect_uris',
-                    models.TextField(blank=True, help_text='Allowed URIs list, space separated', validators=[oauth2_provider.validators.validate_uris]),
+                    models.TextField(blank=True, help_text='Allowed URIs list, space separated'),
                 ),
                 ('client_type', models.CharField(choices=[('confidential', 'Confidential'), ('public', 'Public')], max_length=32)),
                 (

awx/settings/defaults.py

@@ -273,8 +273,8 @@ TEMPLATES = [
     {
         'NAME': 'default',
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'APP_DIRS': True,
         'OPTIONS': {
-            'debug': DEBUG,
             'context_processors': [  # NOQA
                 'django.contrib.auth.context_processors.auth',
                 'django.template.context_processors.debug',
@@ -289,13 +289,10 @@ TEMPLATES = [
                 'social_django.context_processors.backends',
                 'social_django.context_processors.login_redirect',
             ],
-            'loaders': [
-                ('django.template.loaders.cached.Loader', ('django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader'))
-            ],
             'builtins': ['awx.main.templatetags.swagger'],
         },
         'DIRS': [os.path.join(BASE_DIR, 'templates'), os.path.join(BASE_DIR, 'ui', 'build'), os.path.join(BASE_DIR, 'ui', 'public')],
-    }
+    },
 ]
 
 ROOT_URLCONF = 'awx.urls'

awx/settings/development.py

@@ -45,10 +45,6 @@ SESSION_COOKIE_SECURE = False
 # Disallow sending csrf cookies over insecure connections
 CSRF_COOKIE_SECURE = False
 
-# Override django.template.loaders.cached.Loader in defaults.py
-template = next((tpl_backend for tpl_backend in TEMPLATES if tpl_backend['NAME'] == 'default'), None)  # noqa
-template['OPTIONS']['loaders'] = ('django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader')
-
 # Disable Pendo on the UI for development/test.
 # Note: This setting may be overridden by database settings.
 PENDO_TRACKING_STATE = "off"

awx/sso/fields.py

@@ -4,12 +4,13 @@ import inspect
 import json
 import re
 
+import six
+
 # Python LDAP
 import ldap
 import awx
 
 # Django
-from django.utils import six
 from django.utils.translation import ugettext_lazy as _
 
 # Django Auth LDAP

docs/licenses/pkgconfig.txt

@@ -1,19 +0,0 @@
-Copyright (c) 2013 Matthias Vogelgesang <matthias.vogelgesang@gmail.com>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

docs/licenses/ruamel.yaml.clib.txt

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2019 Anthon van der Neut, Ruamel bvba
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

requirements/README.md

@@ -58,7 +58,7 @@ Make sure to delete the old tarball if it is an upgrade.
 Anything pinned in `*.in` files involves additional manual work in
 order to upgrade. Some information related to that work is outlined here.
 
-### django
+### Django
 
 For any upgrade of Django, it must be confirmed that
 we don't regress on FIPS support before merging.
@@ -90,13 +90,10 @@ that we have the latest version
 
 ### django-oauth-toolkit
 
-Version 1.2.0 of this project has a bug that error when revoking tokens.
+Versions later than 1.4.1 throw an error about id_token_id, due to the
-This is fixed in the master branch but is not yet released.
+OpenID Connect work that was done in
-
+https://github.com/jazzband/django-oauth-toolkit/pull/915.  This may
-When upgrading past 1.2.0 in the future, the `0025` migration needs to be
+be fixable by creating a migration on our end?
-edited, just like the old migration was edited in the project:
-https://github.com/jazzband/django-oauth-toolkit/commit/96538876d0d7ea0319ba5286f9bde842a906e1c5
-The field can simply have the validator method `validate_uris` removed.
 
 ### azure-keyvault
 
@@ -117,7 +114,7 @@ https://github.com/adamchainz/django-jsonfield/pull/14
 
 This breaks a very large amount of AWX code that assumes these fields
 are returned as dicts. Upgrading this library will require a refactor
-to accomidate this change.
+to accommodate this change.
 
 ### pip and setuptools
 

requirements/requirements.in

@@ -9,14 +9,14 @@ cryptography>=3.2
 Cython<3 # Since the bump to PyYAML 5.4.1 this is now a mandatory dep
 daphne
 distro
-django==2.2.24  # see UPGRADE BLOCKERs
+django==3.0.14  # see UPGRADE BLOCKERs
 django-auth-ldap
 django-cors-headers>=3.5.0
 django-crum
 django-extensions>=2.2.9  # https://github.com/ansible/awx/pull/6441
-django-guid==2.2.0  # pinned to match Django 2.2
+django-guid==2.2.1  # see https://pypi.org/project/django-guid/ for supported versions
-django-jsonfield==1.2.0  # see UPGRADE BLOCKERs
+django-jsonfield==1.4.1
-django-oauth-toolkit==1.1.3  # see UPGRADE BLOCKERs
+django-oauth-toolkit==1.4.1
 django-polymorphic
 django-pglocks
 django-qsstats-magic
@@ -40,7 +40,7 @@ psycopg2
 psutil
 pygerduty
 pyparsing
-python3-saml
+python3-saml==1.13.0
 python-dsv-sdk
 python-tss-sdk==1.0.0
 python-ldap>=3.3.1 # https://github.com/python-ldap/python-ldap/issues/270

requirements/requirements.txt

@@ -14,6 +14,7 @@ asgiref==3.2.5
     #   channels
     #   channels-redis
     #   daphne
+    #   django
 async-timeout==3.0.1
     # via
     #   aiohttp
@@ -80,13 +81,12 @@ dataclasses==0.6
 defusedxml==0.6.0
     # via
     #   python3-openid
-    #   python3-saml
     #   social-auth-core
 dictdiffer==0.8.1
     # via openshift
 distro==1.5.0
     # via -r /awx_devel/requirements/requirements.in
-django==2.2.24
+django==3.0.14
     # via
     #   -r /awx_devel/requirements/requirements.in
     #   channels
@@ -107,11 +107,11 @@ django-crum==0.7.5
     # via -r /awx_devel/requirements/requirements.in
 django-extensions==2.2.9
     # via -r /awx_devel/requirements/requirements.in
-django-guid==2.2.0
+django-guid==2.2.1
     # via -r /awx_devel/requirements/requirements.in
-django-jsonfield==1.2.0
+django-jsonfield==1.4.1
     # via -r /awx_devel/requirements/requirements.in
-django-oauth-toolkit==1.1.3
+django-oauth-toolkit==1.4.1
     # via -r /awx_devel/requirements/requirements.in
 django-pglocks==1.0.4
     # via -r /awx_devel/requirements/requirements.in
@@ -159,7 +159,7 @@ incremental==17.5.0
     # via twisted
 irc==18.0.0
     # via -r /awx_devel/requirements/requirements.in
-isodate==0.6.0
+isodate==0.6.1
     # via
     #   msrest
     #   python3-saml
@@ -180,7 +180,7 @@ jaraco.text==3.2.0
     # via
     #   irc
     #   jaraco.collections
-jinja2==2.11.2
+jinja2==3.0.3
     # via
     #   -r /awx_devel/requirements/requirements.in
     #   openshift
@@ -192,11 +192,13 @@ kubernetes==11.0.0
     # via openshift
 lockfile==0.12.2
     # via python-daemon
-lxml==4.6.3
+lxml==4.7.0
-    # via xmlsec
+    # via
+    #   python3-saml
+    #   xmlsec
 markdown==3.2.1
     # via -r /awx_devel/requirements/requirements.in
-markupsafe==1.1.1
+markupsafe==2.0.1
     # via jinja2
 more-itertools==8.2.0
     # via
@@ -232,8 +234,6 @@ pexpect==4.7.0
     # via
     #   -r /awx_devel/requirements/requirements.in
     #   ansible-runner
-pkgconfig==1.5.1
-    # via xmlsec
 prometheus-client==0.7.1
     # via -r /awx_devel/requirements/requirements.in
 psutil==5.8.0
@@ -293,7 +293,7 @@ python-tss-sdk==1.0.0
     # via -r /awx_devel/requirements/requirements.in
 python3-openid==3.1.0
     # via social-auth-core
-python3-saml==1.9.0
+python3-saml==1.13.0
     # via -r /awx_devel/requirements/requirements.in
 pytz==2019.3
     # via
@@ -336,8 +336,6 @@ rsa==4.7.2
     # via google-auth
 ruamel.yaml==0.16.10
     # via openshift
-ruamel.yaml.clib==0.2.0
-    # via ruamel.yaml
 schedule==0.6.0
     # via -r /awx_devel/requirements/requirements.in
 service-identity==18.1.0
@@ -348,6 +346,7 @@ six==1.14.0
     #   automat
     #   cryptography
     #   django-extensions
+    #   django-jsonfield
     #   django-pglocks
     #   google-auth
     #   isodate
@@ -407,7 +406,7 @@ websocket-client==0.57.0
     # via kubernetes
 wheel==0.36.2
     # via -r /awx_devel/requirements/requirements.in
-xmlsec==1.3.3
+xmlsec==1.3.12
     # via python3-saml
 yarl==1.4.2
     # via aiohttp

requirements/requirements_dev.txt

@@ -1,4 +1,4 @@
-django-debug-toolbar==1.11.1
+django-debug-toolbar==3.2.4
 django-rest-swagger
 # pprofile - re-add once https://github.com/vpelletier/pprofile/issues/41 is addressed
 ipython==7.21.0

requirements/updater.sh

@@ -18,7 +18,8 @@ generate_requirements() {
   # shellcheck disable=SC1090
   source ${venv}/bin/activate
 
-  ${venv}/bin/python3 -m pip install -U pip pip-tools
+  # FIXME: https://github.com/jazzband/pip-tools/issues/1558
+  ${venv}/bin/python3 -m pip install -U 'pip<22.0' pip-tools
 
   ${pip_compile} "${requirements_in}" "${requirements_git}" --output-file requirements.txt
   # consider the git requirements for purposes of resolving deps