From df61d1a59ce6c9142e8f0848a24b6f38e7b0be94 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 31 Jan 2022 11:40:00 -0500 Subject: [PATCH] Upgrade to Django 3.0 - upgrades - Django 3.0.14 - django-jsonfield 1.4.1 (from 1.2.0) - django-oauth-toolkit 1.4.1 (from 1.1.3) - Stopping here because later versions have changes to the underlying model to support OpenID Connect. Presumably this can be dealt with via a migration in our project. - django-guid 2.2.1 (from 2.2.0) - django-debug-toolbar 3.2.4 (from 1.11.1) - python3-saml 1.13.0 (from 1.9.0) - xmlsec 1.3.12 (from 1.3.3) - Remove our project's use of django.utils.six in favor of directly using six, in awx.sso.fields. - Temporarily monkey patch six back in as django.utils.six, since django-jsonfield makes use of that import, and is no longer being updated. Hopefully we can do away with this dependency with the new generalized JSONField brought in with Django 3.1. - Force a json decoder to be used with all instances of JSONField brought in by django-jsonfield. This deals with the 'cast to text' problem noted previously in our UPGRADE_BLOCKERS. - Remove the validate_uris validator from the OAuth2Application in migration 0025, per the UPGRADE_BLOCKERS, and remove that note. - Update the TEMPLATES setting to satisfy Django Debug Toolbar. It requires at least one entry that has APP_DIRS=True, and as near as I can tell our custom OPTIONS.loaders setting was effectively doing the same thing as Django's own machinery if this setting is set. --- awx/__init__.py | 4 +++ awx/main/fields.py | 4 +++ ...330_add_oauth_activity_stream_registrar.py | 2 +- awx/settings/defaults.py | 7 ++--- awx/settings/development.py | 4 --- awx/sso/fields.py | 3 +- docs/licenses/pkgconfig.txt | 19 ------------ docs/licenses/ruamel.yaml.clib.txt | 21 ------------- requirements/README.md | 15 ++++----- requirements/requirements.in | 10 +++--- requirements/requirements.txt | 31 +++++++++---------- requirements/requirements_dev.txt | 2 +- requirements/updater.sh | 3 +- 13 files changed, 42 insertions(+), 83 deletions(-) delete mode 100644 docs/licenses/pkgconfig.txt delete mode 100644 docs/licenses/ruamel.yaml.clib.txt diff --git a/awx/__init__.py b/awx/__init__.py index fa3e164092..31806538c2 100644 --- a/awx/__init__.py +++ b/awx/__init__.py @@ -6,6 +6,8 @@ import os import sys import warnings +import six + from pkg_resources import get_distribution __version__ = get_distribution('awx').version @@ -35,7 +37,9 @@ else: from django.db.models import indexes from django.db.backends.utils import names_digest from django.db import connection + from django import utils + utils.six = six # FIXME: monkey patch to get us through for now if HAS_DJANGO is True: diff --git a/awx/main/fields.py b/awx/main/fields.py index 95ebfbca73..71bf0c612f 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -72,6 +72,10 @@ Draft4Validator.VALIDATORS['enum'] = __enum_validate__ class JSONField(upstream_JSONField): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.decoder_kwargs = {'cls': json.JSONDecoder} # FIXME + def db_type(self, connection): return 'text' diff --git a/awx/main/migrations/0025_v330_add_oauth_activity_stream_registrar.py b/awx/main/migrations/0025_v330_add_oauth_activity_stream_registrar.py index cc1d1bfeba..e26571f1b9 100644 --- a/awx/main/migrations/0025_v330_add_oauth_activity_stream_registrar.py +++ b/awx/main/migrations/0025_v330_add_oauth_activity_stream_registrar.py @@ -29,7 +29,7 @@ class Migration(migrations.Migration): ('client_id', models.CharField(db_index=True, default=oauth2_provider.generators.generate_client_id, max_length=100, unique=True)), ( 'redirect_uris', - models.TextField(blank=True, help_text='Allowed URIs list, space separated', validators=[oauth2_provider.validators.validate_uris]), + models.TextField(blank=True, help_text='Allowed URIs list, space separated'), ), ('client_type', models.CharField(choices=[('confidential', 'Confidential'), ('public', 'Public')], max_length=32)), ( diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index bc3c2549c3..fcd422a3dd 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -273,8 +273,8 @@ TEMPLATES = [ { 'NAME': 'default', 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'APP_DIRS': True, 'OPTIONS': { - 'debug': DEBUG, 'context_processors': [ # NOQA 'django.contrib.auth.context_processors.auth', 'django.template.context_processors.debug', @@ -289,13 +289,10 @@ TEMPLATES = [ 'social_django.context_processors.backends', 'social_django.context_processors.login_redirect', ], - 'loaders': [ - ('django.template.loaders.cached.Loader', ('django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader')) - ], 'builtins': ['awx.main.templatetags.swagger'], }, 'DIRS': [os.path.join(BASE_DIR, 'templates'), os.path.join(BASE_DIR, 'ui', 'build'), os.path.join(BASE_DIR, 'ui', 'public')], - } + }, ] ROOT_URLCONF = 'awx.urls' diff --git a/awx/settings/development.py b/awx/settings/development.py index 70b64643dd..be1c115606 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -45,10 +45,6 @@ SESSION_COOKIE_SECURE = False # Disallow sending csrf cookies over insecure connections CSRF_COOKIE_SECURE = False -# Override django.template.loaders.cached.Loader in defaults.py -template = next((tpl_backend for tpl_backend in TEMPLATES if tpl_backend['NAME'] == 'default'), None) # noqa -template['OPTIONS']['loaders'] = ('django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader') - # Disable Pendo on the UI for development/test. # Note: This setting may be overridden by database settings. PENDO_TRACKING_STATE = "off" diff --git a/awx/sso/fields.py b/awx/sso/fields.py index e2d46d9362..0e4af005c9 100644 --- a/awx/sso/fields.py +++ b/awx/sso/fields.py @@ -4,12 +4,13 @@ import inspect import json import re +import six + # Python LDAP import ldap import awx # Django -from django.utils import six from django.utils.translation import ugettext_lazy as _ # Django Auth LDAP diff --git a/docs/licenses/pkgconfig.txt b/docs/licenses/pkgconfig.txt deleted file mode 100644 index 716f12754d..0000000000 --- a/docs/licenses/pkgconfig.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2013 Matthias Vogelgesang - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/docs/licenses/ruamel.yaml.clib.txt b/docs/licenses/ruamel.yaml.clib.txt deleted file mode 100644 index 1c3e20a20e..0000000000 --- a/docs/licenses/ruamel.yaml.clib.txt +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2019 Anthon van der Neut, Ruamel bvba - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/requirements/README.md b/requirements/README.md index 69ab2d4bcf..5dc2638c3c 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -58,7 +58,7 @@ Make sure to delete the old tarball if it is an upgrade. Anything pinned in `*.in` files involves additional manual work in order to upgrade. Some information related to that work is outlined here. -### django +### Django For any upgrade of Django, it must be confirmed that we don't regress on FIPS support before merging. @@ -90,13 +90,10 @@ that we have the latest version ### django-oauth-toolkit -Version 1.2.0 of this project has a bug that error when revoking tokens. -This is fixed in the master branch but is not yet released. - -When upgrading past 1.2.0 in the future, the `0025` migration needs to be -edited, just like the old migration was edited in the project: -https://github.com/jazzband/django-oauth-toolkit/commit/96538876d0d7ea0319ba5286f9bde842a906e1c5 -The field can simply have the validator method `validate_uris` removed. +Versions later than 1.4.1 throw an error about id_token_id, due to the +OpenID Connect work that was done in +https://github.com/jazzband/django-oauth-toolkit/pull/915. This may +be fixable by creating a migration on our end? ### azure-keyvault @@ -117,7 +114,7 @@ https://github.com/adamchainz/django-jsonfield/pull/14 This breaks a very large amount of AWX code that assumes these fields are returned as dicts. Upgrading this library will require a refactor -to accomidate this change. +to accommodate this change. ### pip and setuptools diff --git a/requirements/requirements.in b/requirements/requirements.in index 94ece3085d..d60639aa1b 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -9,14 +9,14 @@ cryptography>=3.2 Cython<3 # Since the bump to PyYAML 5.4.1 this is now a mandatory dep daphne distro -django==2.2.24 # see UPGRADE BLOCKERs +django==3.0.14 # see UPGRADE BLOCKERs django-auth-ldap django-cors-headers>=3.5.0 django-crum django-extensions>=2.2.9 # https://github.com/ansible/awx/pull/6441 -django-guid==2.2.0 # pinned to match Django 2.2 -django-jsonfield==1.2.0 # see UPGRADE BLOCKERs -django-oauth-toolkit==1.1.3 # see UPGRADE BLOCKERs +django-guid==2.2.1 # see https://pypi.org/project/django-guid/ for supported versions +django-jsonfield==1.4.1 +django-oauth-toolkit==1.4.1 django-polymorphic django-pglocks django-qsstats-magic @@ -40,7 +40,7 @@ psycopg2 psutil pygerduty pyparsing -python3-saml +python3-saml==1.13.0 python-dsv-sdk python-tss-sdk==1.0.0 python-ldap>=3.3.1 # https://github.com/python-ldap/python-ldap/issues/270 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f05ebe3b96..2e16a46797 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -14,6 +14,7 @@ asgiref==3.2.5 # channels # channels-redis # daphne + # django async-timeout==3.0.1 # via # aiohttp @@ -80,13 +81,12 @@ dataclasses==0.6 defusedxml==0.6.0 # via # python3-openid - # python3-saml # social-auth-core dictdiffer==0.8.1 # via openshift distro==1.5.0 # via -r /awx_devel/requirements/requirements.in -django==2.2.24 +django==3.0.14 # via # -r /awx_devel/requirements/requirements.in # channels @@ -107,11 +107,11 @@ django-crum==0.7.5 # via -r /awx_devel/requirements/requirements.in django-extensions==2.2.9 # via -r /awx_devel/requirements/requirements.in -django-guid==2.2.0 +django-guid==2.2.1 # via -r /awx_devel/requirements/requirements.in -django-jsonfield==1.2.0 +django-jsonfield==1.4.1 # via -r /awx_devel/requirements/requirements.in -django-oauth-toolkit==1.1.3 +django-oauth-toolkit==1.4.1 # via -r /awx_devel/requirements/requirements.in django-pglocks==1.0.4 # via -r /awx_devel/requirements/requirements.in @@ -159,7 +159,7 @@ incremental==17.5.0 # via twisted irc==18.0.0 # via -r /awx_devel/requirements/requirements.in -isodate==0.6.0 +isodate==0.6.1 # via # msrest # python3-saml @@ -180,7 +180,7 @@ jaraco.text==3.2.0 # via # irc # jaraco.collections -jinja2==2.11.2 +jinja2==3.0.3 # via # -r /awx_devel/requirements/requirements.in # openshift @@ -192,11 +192,13 @@ kubernetes==11.0.0 # via openshift lockfile==0.12.2 # via python-daemon -lxml==4.6.3 - # via xmlsec +lxml==4.7.0 + # via + # python3-saml + # xmlsec markdown==3.2.1 # via -r /awx_devel/requirements/requirements.in -markupsafe==1.1.1 +markupsafe==2.0.1 # via jinja2 more-itertools==8.2.0 # via @@ -232,8 +234,6 @@ pexpect==4.7.0 # via # -r /awx_devel/requirements/requirements.in # ansible-runner -pkgconfig==1.5.1 - # via xmlsec prometheus-client==0.7.1 # via -r /awx_devel/requirements/requirements.in psutil==5.8.0 @@ -293,7 +293,7 @@ python-tss-sdk==1.0.0 # via -r /awx_devel/requirements/requirements.in python3-openid==3.1.0 # via social-auth-core -python3-saml==1.9.0 +python3-saml==1.13.0 # via -r /awx_devel/requirements/requirements.in pytz==2019.3 # via @@ -336,8 +336,6 @@ rsa==4.7.2 # via google-auth ruamel.yaml==0.16.10 # via openshift -ruamel.yaml.clib==0.2.0 - # via ruamel.yaml schedule==0.6.0 # via -r /awx_devel/requirements/requirements.in service-identity==18.1.0 @@ -348,6 +346,7 @@ six==1.14.0 # automat # cryptography # django-extensions + # django-jsonfield # django-pglocks # google-auth # isodate @@ -407,7 +406,7 @@ websocket-client==0.57.0 # via kubernetes wheel==0.36.2 # via -r /awx_devel/requirements/requirements.in -xmlsec==1.3.3 +xmlsec==1.3.12 # via python3-saml yarl==1.4.2 # via aiohttp diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt index 582eac7fb9..bea1e88dc2 100644 --- a/requirements/requirements_dev.txt +++ b/requirements/requirements_dev.txt @@ -1,4 +1,4 @@ -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 django-rest-swagger # pprofile - re-add once https://github.com/vpelletier/pprofile/issues/41 is addressed ipython==7.21.0 diff --git a/requirements/updater.sh b/requirements/updater.sh index fa9ae7ddd2..01f6000d2c 100755 --- a/requirements/updater.sh +++ b/requirements/updater.sh @@ -18,7 +18,8 @@ generate_requirements() { # shellcheck disable=SC1090 source ${venv}/bin/activate - ${venv}/bin/python3 -m pip install -U pip pip-tools + # FIXME: https://github.com/jazzband/pip-tools/issues/1558 + ${venv}/bin/python3 -m pip install -U 'pip<22.0' pip-tools ${pip_compile} "${requirements_in}" "${requirements_git}" --output-file requirements.txt # consider the git requirements for purposes of resolving deps