diff --git a/awx/conf/migrations/0001_initial.py b/awx/conf/migrations/0001_initial.py index b239f5e143..ffde1c4bd6 100644 --- a/awx/conf/migrations/0001_initial.py +++ b/awx/conf/migrations/0001_initial.py @@ -4,6 +4,8 @@ from __future__ import unicode_literals from django.db import migrations, models from django.conf import settings +import awx.main.fields + class Migration(migrations.Migration): @@ -17,7 +19,7 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(default=None, editable=False)), ('modified', models.DateTimeField(default=None, editable=False)), ('key', models.CharField(max_length=255)), - ('value', models.JSONField(null=True)), + ('value', awx.main.fields.JSONBlob(null=True)), ( 'user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True), diff --git a/awx/conf/migrations/0003_v310_JSONField_changes.py b/awx/conf/migrations/0003_v310_JSONField_changes.py index d312c40b1d..6c0b5cba98 100644 --- a/awx/conf/migrations/0003_v310_JSONField_changes.py +++ b/awx/conf/migrations/0003_v310_JSONField_changes.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from django.db import migrations, models +from django.db import migrations + +import awx.main.fields class Migration(migrations.Migration): dependencies = [('conf', '0002_v310_copy_tower_settings')] - operations = [migrations.AlterField(model_name='setting', name='value', field=models.JSONField(null=True))] + operations = [migrations.AlterField(model_name='setting', name='value', field=awx.main.fields.JSONBlob(null=True))] diff --git a/awx/conf/models.py b/awx/conf/models.py index 05162436d1..79bc572d57 100644 --- a/awx/conf/models.py +++ b/awx/conf/models.py @@ -8,6 +8,7 @@ import json from django.db import models # AWX +from awx.main.fields import JSONBlob from awx.main.models.base import CreatedModifiedModel, prevent_search from awx.main.utils import encrypt_field from awx.conf import settings_registry @@ -18,7 +19,7 @@ __all__ = ['Setting'] class Setting(CreatedModifiedModel): key = models.CharField(max_length=255) - value = models.JSONField(null=True) + value = JSONBlob(null=True) user = prevent_search(models.ForeignKey('auth.User', related_name='settings', default=None, null=True, editable=False, on_delete=models.CASCADE)) def __str__(self): diff --git a/awx/main/fields.py b/awx/main/fields.py index 83ab57f37d..3372627f91 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -43,7 +43,6 @@ from rest_framework import serializers from awx.main.utils.filters import SmartFilter from awx.main.utils.encryption import encrypt_value, decrypt_value, get_encryption_key from awx.main.validators import validate_ssh_private_key -from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role, ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR from awx.main.constants import ENV_BLOCKLIST from awx.main import utils @@ -130,6 +129,9 @@ def is_implicit_parent(parent_role, child_role): the model definition. This does not include any role parents that might have been set by the user. """ + # Avoid circular import + from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR + if child_role.content_object is None: # The only singleton implicit parent is the system admin being # a parent of the system auditor role @@ -286,6 +288,9 @@ class ImplicitRoleField(models.ForeignKey): Model = utils.get_current_apps().get_model('main', instance.__class__.__name__) latest_instance = Model.objects.get(pk=instance.pk) + # Avoid circular import + from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role + with batch_role_ancestor_rebuilding(): # Create any missing role objects missing_roles = [] @@ -340,6 +345,10 @@ class ImplicitRoleField(models.ForeignKey): Role_ = utils.get_current_apps().get_model('main', 'Role') child_ids = [x for x in Role_.parents.through.objects.filter(to_role_id__in=role_ids).distinct().values_list('from_role_id', flat=True)] Role_.objects.filter(id__in=role_ids).delete() + + # Avoid circular import + from awx.main.models.rbac import Role + Role.rebuild_role_ancestor_list([], child_ids) diff --git a/awx/main/migrations/0001_initial.py b/awx/main/migrations/0001_initial.py index c3dcbe36b7..aa78bdca16 100644 --- a/awx/main/migrations/0001_initial.py +++ b/awx/main/migrations/0001_initial.py @@ -622,7 +622,7 @@ class Migration(migrations.Migration): ('dtend', models.DateTimeField(default=None, null=True, editable=False)), ('rrule', models.CharField(max_length=255)), ('next_run', models.DateTimeField(default=None, null=True, editable=False)), - ('extra_data', models.JSONField(default=dict, null=True, blank=True)), + ('extra_data', awx.main.fields.JSONBlob(default=dict, blank=True)), ( 'created_by', models.ForeignKey( @@ -750,7 +750,7 @@ class Migration(migrations.Migration): ('elapsed', models.DecimalField(editable=False, max_digits=12, decimal_places=3)), ('job_args', models.TextField(default='', editable=False, blank=True)), ('job_cwd', models.CharField(default='', max_length=1024, editable=False, blank=True)), - ('job_env', models.JSONField(default=dict, editable=False, null=True, blank=True)), + ('job_env', awx.main.fields.JSONBlob(default=dict, editable=False, blank=True)), ('job_explanation', models.TextField(default='', editable=False, blank=True)), ('start_args', models.TextField(default='', editable=False, blank=True)), ('result_stdout_text', models.TextField(default='', editable=False, blank=True)), @@ -1034,7 +1034,7 @@ class Migration(migrations.Migration): ('host_config_key', models.CharField(default='', max_length=1024, blank=True)), ('ask_variables_on_launch', models.BooleanField(default=False)), ('survey_enabled', models.BooleanField(default=False)), - ('survey_spec', models.JSONField(default=dict, blank=True)), + ('survey_spec', awx.main.fields.JSONBlob(default=dict, blank=True)), ], options={ 'ordering': ('name',), diff --git a/awx/main/migrations/0002_squashed_v300_release.py b/awx/main/migrations/0002_squashed_v300_release.py index 5f23ed566f..8093de7175 100644 --- a/awx/main/migrations/0002_squashed_v300_release.py +++ b/awx/main/migrations/0002_squashed_v300_release.py @@ -198,7 +198,7 @@ class Migration(migrations.Migration): ), ('recipients', models.TextField(default='', editable=False, blank=True)), ('subject', models.TextField(default='', editable=False, blank=True)), - ('body', models.JSONField(default=dict, null=True, blank=True)), + ('body', awx.main.fields.JSONBlob(default=dict, blank=True)), ], options={ 'ordering': ('pk',), @@ -229,7 +229,7 @@ class Migration(migrations.Migration): ], ), ), - ('notification_configuration', models.JSONField(default=dict)), + ('notification_configuration', awx.main.fields.JSONBlob(default=dict)), ( 'created_by', models.ForeignKey( diff --git a/awx/main/migrations/0004_squashed_v310_release.py b/awx/main/migrations/0004_squashed_v310_release.py index c0ac0d4a04..b1d45d10d6 100644 --- a/awx/main/migrations/0004_squashed_v310_release.py +++ b/awx/main/migrations/0004_squashed_v310_release.py @@ -220,7 +220,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='workflowjobnode', name='char_prompts', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AddField( model_name='workflowjobnode', @@ -259,7 +259,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='workflowjobtemplatenode', name='char_prompts', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AddField( model_name='workflowjobtemplatenode', @@ -307,12 +307,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='job', name='artifacts', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), migrations.AddField( model_name='workflowjobnode', name='ancestor_artifacts', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), # Job timeout settings migrations.AddField( @@ -380,7 +380,9 @@ class Migration(migrations.Migration): migrations.AddField( model_name='project', name='playbook_files', - field=models.JSONField(default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True), + field=awx.main.fields.JSONBlob( + default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True + ), ), # Job events to stdout migrations.AddField( @@ -536,7 +538,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='workflowjob', name='survey_passwords', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), migrations.AddField( model_name='workflowjobtemplate', @@ -546,7 +548,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='workflowjobtemplate', name='survey_spec', - field=models.JSONField(default=dict, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), # JSON field changes migrations.AlterField( @@ -557,12 +559,12 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='job', name='artifacts', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), migrations.AlterField( model_name='job', name='survey_passwords', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), migrations.AlterField( model_name='jobevent', @@ -572,57 +574,59 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='jobtemplate', name='survey_spec', - field=models.JSONField(default=dict, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AlterField( model_name='notification', name='body', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AlterField( model_name='notificationtemplate', name='notification_configuration', - field=models.JSONField(default=dict), + field=awx.main.fields.JSONBlob(default=dict), ), migrations.AlterField( model_name='project', name='playbook_files', - field=models.JSONField(default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True), + field=awx.main.fields.JSONBlob( + default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True + ), ), migrations.AlterField( model_name='schedule', name='extra_data', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AlterField( model_name='unifiedjob', name='job_env', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), migrations.AlterField( model_name='workflowjob', name='survey_passwords', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), migrations.AlterField( model_name='workflowjobnode', name='ancestor_artifacts', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), migrations.AlterField( model_name='workflowjobnode', name='char_prompts', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AlterField( model_name='workflowjobtemplate', name='survey_spec', - field=models.JSONField(default=dict, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AlterField( model_name='workflowjobtemplatenode', name='char_prompts', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), # Job Project Update migrations.AddField( diff --git a/awx/main/migrations/0006_v320_release.py b/awx/main/migrations/0006_v320_release.py index c05bee3eec..0f88577baa 100644 --- a/awx/main/migrations/0006_v320_release.py +++ b/awx/main/migrations/0006_v320_release.py @@ -175,7 +175,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='project', name='inventory_files', - field=models.JSONField( + field=awx.main.fields.JSONBlob( default=list, help_text='Suggested list of content that could be Ansible inventory in the project', verbose_name='Inventory Files', diff --git a/awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py b/awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py index 56c86b19a8..f90e5a966b 100644 --- a/awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py +++ b/awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from django.db import migrations, models +from django.db import migrations + +import awx.main.fields class Migration(migrations.Migration): @@ -14,6 +16,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='activitystream', name='setting', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), ] diff --git a/awx/main/migrations/0014_v330_saved_launchtime_configs.py b/awx/main/migrations/0014_v330_saved_launchtime_configs.py index 38c5d2b2f6..fdefdcaed8 100644 --- a/awx/main/migrations/0014_v330_saved_launchtime_configs.py +++ b/awx/main/migrations/0014_v330_saved_launchtime_configs.py @@ -20,7 +20,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='schedule', name='char_prompts', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AddField( model_name='schedule', @@ -37,7 +37,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='schedule', name='survey_passwords', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), migrations.AddField( model_name='workflowjobnode', @@ -47,12 +47,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='workflowjobnode', name='extra_data', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AddField( model_name='workflowjobnode', name='survey_passwords', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), migrations.AddField( model_name='workflowjobtemplatenode', @@ -62,12 +62,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='workflowjobtemplatenode', name='extra_data', - field=models.JSONField(default=dict, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, blank=True), ), migrations.AddField( model_name='workflowjobtemplatenode', name='survey_passwords', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), # Run data migration before removing the old credential field migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrations.RunPython.noop), @@ -85,9 +85,9 @@ class Migration(migrations.Migration): name='JobLaunchConfig', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('extra_data', models.JSONField(blank=True, null=True, default=dict)), - ('survey_passwords', models.JSONField(blank=True, null=True, default=dict, editable=False)), - ('char_prompts', models.JSONField(blank=True, null=True, default=dict)), + ('extra_data', awx.main.fields.JSONBlob(blank=True, default=dict)), + ('survey_passwords', awx.main.fields.JSONBlob(blank=True, default=dict, editable=False)), + ('char_prompts', awx.main.fields.JSONBlob(blank=True, default=dict)), ('credentials', models.ManyToManyField(related_name='joblaunchconfigs', to='main.Credential')), ( 'inventory', diff --git a/awx/main/migrations/0020_v330_instancegroup_policies.py b/awx/main/migrations/0020_v330_instancegroup_policies.py index 0577f14ee9..a6275d4820 100644 --- a/awx/main/migrations/0020_v330_instancegroup_policies.py +++ b/awx/main/migrations/0020_v330_instancegroup_policies.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +from decimal import Decimal from django.db import migrations, models -from decimal import Decimal + +import awx.main.fields class Migration(migrations.Migration): @@ -15,7 +17,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='instancegroup', name='policy_instance_list', - field=models.JSONField( + field=awx.main.fields.JSONBlob( default=list, help_text='List of exact-match Instances that will always be automatically assigned to this group', blank=True ), ), diff --git a/awx/main/migrations/0038_v330_add_deleted_activitystream_actor.py b/awx/main/migrations/0038_v330_add_deleted_activitystream_actor.py index 504fa14eb3..6e921a9b40 100644 --- a/awx/main/migrations/0038_v330_add_deleted_activitystream_actor.py +++ b/awx/main/migrations/0038_v330_add_deleted_activitystream_actor.py @@ -2,7 +2,9 @@ # Generated by Django 1.11.11 on 2018-05-21 19:51 from __future__ import unicode_literals -from django.db import models, migrations +from django.db import migrations + +import awx.main.fields class Migration(migrations.Migration): @@ -15,6 +17,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='activitystream', name='deleted_actor', - field=models.JSONField(null=True), + field=awx.main.fields.JSONBlob(null=True), ), ] diff --git a/awx/main/migrations/0053_v340_workflow_inventory.py b/awx/main/migrations/0053_v340_workflow_inventory.py index e3dd56a3b2..7e4b7590ec 100644 --- a/awx/main/migrations/0053_v340_workflow_inventory.py +++ b/awx/main/migrations/0053_v340_workflow_inventory.py @@ -2,10 +2,11 @@ # Generated by Django 1.11.11 on 2018-09-27 19:50 from __future__ import unicode_literals -import awx.main.fields from django.db import migrations, models import django.db.models.deletion +import awx.main.fields + class Migration(migrations.Migration): @@ -17,7 +18,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='workflowjob', name='char_prompts', - field=models.JSONField(blank=True, null=True, default=dict), + field=awx.main.fields.JSONBlob(blank=True, default=dict), ), migrations.AddField( model_name='workflowjob', diff --git a/awx/main/migrations/0085_v360_add_notificationtemplate_messages.py b/awx/main/migrations/0085_v360_add_notificationtemplate_messages.py index c2c69bb440..7e34b87ffe 100644 --- a/awx/main/migrations/0085_v360_add_notificationtemplate_messages.py +++ b/awx/main/migrations/0085_v360_add_notificationtemplate_messages.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals from django.db import migrations, models +import awx.main.fields import awx.main.models.notifications @@ -17,7 +18,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='notificationtemplate', name='messages', - field=models.JSONField( + field=awx.main.fields.JSONBlob( default=awx.main.models.notifications.NotificationTemplate.default_messages, help_text='Optional custom messages for notification template.', null=True, diff --git a/awx/main/migrations/0090_v360_WFJT_prompts.py b/awx/main/migrations/0090_v360_WFJT_prompts.py index fdc3b85fcc..d8b4a06073 100644 --- a/awx/main/migrations/0090_v360_WFJT_prompts.py +++ b/awx/main/migrations/0090_v360_WFJT_prompts.py @@ -24,7 +24,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='workflowjobtemplate', name='char_prompts', - field=models.JSONField(blank=True, null=True, default=dict), + field=awx.main.fields.JSONBlob(blank=True, default=dict), ), migrations.AlterField( model_name='joblaunchconfig', diff --git a/awx/main/migrations/_squashed_30.py b/awx/main/migrations/_squashed_30.py index 90c2dd061b..c63e9915ec 100644 --- a/awx/main/migrations/_squashed_30.py +++ b/awx/main/migrations/_squashed_30.py @@ -29,7 +29,7 @@ SQUASHED_30 = { migrations.AddField( model_name='job', name='survey_passwords', - field=models.JSONField(default=dict, editable=False, null=True, blank=True), + field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True), ), ], '0031_v302_migrate_survey_passwords': [ diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 107c7a9418..ed49b98083 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -3,7 +3,6 @@ # Django from django.conf import settings # noqa -from django.db import connection from django.db.models.signals import pre_delete # noqa # AWX @@ -98,93 +97,6 @@ User.add_to_class('can_access_with_errors', check_user_access_with_errors) User.add_to_class('accessible_objects', user_accessible_objects) -def convert_jsonfields_to_jsonb(): - if connection.vendor != 'postgresql': - return - - # fmt: off - fields = [ # Table name, expensive or not, tuple of column names - ('conf_setting', False, ( - 'value', - )), - ('main_instancegroup', False, ( - 'policy_instance_list', - )), - ('main_jobtemplate', False, ( - 'survey_spec', - )), - ('main_notificationtemplate', False, ( - 'notification_configuration', - 'messages', - )), - ('main_project', False, ( - 'playbook_files', - 'inventory_files', - )), - ('main_schedule', False, ( - 'extra_data', - 'char_prompts', - 'survey_passwords', - )), - ('main_workflowjobtemplate', False, ( - 'survey_spec', - 'char_prompts', - )), - ('main_workflowjobtemplatenode', False, ( - 'char_prompts', - 'extra_data', - 'survey_passwords', - )), - ('main_activitystream', True, ( - 'setting', # NN = NOT NULL - 'deleted_actor', - )), - ('main_job', True, ( - 'survey_passwords', # NN - 'artifacts', # NN - )), - ('main_joblaunchconfig', True, ( - 'extra_data', # NN - 'survey_passwords', # NN - 'char_prompts', # NN - )), - ('main_notification', True, ( - 'body', # NN - )), - ('main_unifiedjob', True, ( - 'job_env', # NN - )), - ('main_workflowjob', True, ( - 'survey_passwords', # NN - 'char_prompts', # NN - )), - ('main_workflowjobnode', True, ( - 'char_prompts', # NN - 'ancestor_artifacts', # NN - 'extra_data', # NN - 'survey_passwords', # NN - )), - ] - # fmt: on - - with connection.cursor() as cursor: - for table, expensive, columns in fields: - cursor.execute( - """ - select count(1) from information_schema.columns - where - table_name = %s and - column_name in %s and - data_type != 'jsonb'; - """, - (table, columns), - ) - if cursor.fetchone()[0]: - from awx.main.tasks.system import migrate_json_fields - - migrate_json_fields.apply_async([table, expensive, columns]) - - def cleanup_created_modified_by(sender, **kwargs): # work around a bug in django-polymorphic that doesn't properly # handle cascades for reverse foreign keys on the polymorphic base model diff --git a/awx/main/models/activity_stream.py b/awx/main/models/activity_stream.py index aa0ab9d9d6..fad08377fd 100644 --- a/awx/main/models/activity_stream.py +++ b/awx/main/models/activity_stream.py @@ -3,6 +3,7 @@ # AWX from awx.api.versioning import reverse +from awx.main.fields import JSONBlob from awx.main.models.base import accepts_json # Django @@ -35,7 +36,7 @@ class ActivityStream(models.Model): operation = models.CharField(max_length=13, choices=OPERATION_CHOICES) timestamp = models.DateTimeField(auto_now_add=True) changes = accepts_json(models.TextField(blank=True)) - deleted_actor = models.JSONField(null=True) + deleted_actor = JSONBlob(null=True) action_node = models.CharField( blank=True, default='', @@ -83,7 +84,7 @@ class ActivityStream(models.Model): o_auth2_application = models.ManyToManyField("OAuth2Application", blank=True) o_auth2_access_token = models.ManyToManyField("OAuth2AccessToken", blank=True) - setting = models.JSONField(default=dict, null=True, blank=True) + setting = JSONBlob(default=dict, blank=True) def __str__(self): operation = self.operation if 'operation' in self.__dict__ else '_delayed_' diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py index 5200f99d3e..f52d1d0d89 100644 --- a/awx/main/models/ha.py +++ b/awx/main/models/ha.py @@ -19,6 +19,7 @@ from solo.models import SingletonModel from awx import __version__ as awx_application_version from awx.api.versioning import reverse +from awx.main.fields import JSONBlob from awx.main.managers import InstanceManager, InstanceGroupManager, UUID_DEFAULT from awx.main.constants import JOB_FOLDER_PREFIX from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search @@ -328,7 +329,7 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin): ) policy_instance_percentage = models.IntegerField(default=0, help_text=_("Percentage of Instances to automatically assign to this group")) policy_instance_minimum = models.IntegerField(default=0, help_text=_("Static minimum number of Instances to automatically assign to this group")) - policy_instance_list = models.JSONField( + policy_instance_list = JSONBlob( default=list, blank=True, help_text=_("List of exact-match Instances that will always be automatically assigned to this group") ) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 2660c69c62..28717192cf 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -44,7 +44,7 @@ from awx.main.models.notifications import ( JobNotificationMixin, ) from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField -from awx.main.fields import ImplicitRoleField, AskForField +from awx.main.fields import ImplicitRoleField, AskForField, JSONBlob from awx.main.models.mixins import ( ResourceMixin, SurveyJobTemplateMixin, @@ -547,9 +547,8 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana editable=False, through='JobHostSummary', ) - artifacts = models.JSONField( + artifacts = JSONBlob( default=dict, - null=True, blank=True, editable=False, ) @@ -887,7 +886,7 @@ class LaunchTimeConfigBase(BaseModel): ) # All standard fields are stored in this dictionary field # This is a solution to the nullable CharField problem, specific to prompting - char_prompts = models.JSONField(default=dict, null=True, blank=True) + char_prompts = JSONBlob(default=dict, blank=True) def prompts_dict(self, display=False): data = {} @@ -940,12 +939,11 @@ class LaunchTimeConfig(LaunchTimeConfigBase): abstract = True # Special case prompting fields, even more special than the other ones - extra_data = models.JSONField(default=dict, null=True, blank=True) + extra_data = JSONBlob(default=dict, blank=True) survey_passwords = prevent_search( - models.JSONField( + JSONBlob( default=dict, editable=False, - null=True, blank=True, ) ) diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index dc144f96ec..8396585bb5 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -24,7 +24,7 @@ from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_lice from awx.main.utils.execution_environments import get_default_execution_environment from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted from awx.main.utils.polymorphic import build_polymorphic_ctypes_map -from awx.main.fields import AskForField +from awx.main.fields import AskForField, JSONBlob from awx.main.constants import ACTIVE_STATES @@ -103,7 +103,7 @@ class SurveyJobTemplateMixin(models.Model): survey_enabled = models.BooleanField( default=False, ) - survey_spec = prevent_search(models.JSONField(default=dict, blank=True)) + survey_spec = prevent_search(JSONBlob(default=dict, blank=True)) ask_variables_on_launch = AskForField(blank=True, default=False, allows_field='extra_vars') def survey_password_variables(self): @@ -365,10 +365,9 @@ class SurveyJobMixin(models.Model): abstract = True survey_passwords = prevent_search( - models.JSONField( + JSONBlob( default=dict, editable=False, - null=True, blank=True, ) ) diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index 9bfd1bc6b5..fb8af5b5f0 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -17,6 +17,7 @@ from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError # AWX from awx.api.versioning import reverse +from awx.main.fields import JSONBlob from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search from awx.main.utils import encrypt_field, decrypt_field, set_environ from awx.main.notifications.email_backend import CustomEmailBackend @@ -69,12 +70,12 @@ class NotificationTemplate(CommonModelNameNotUnique): choices=NOTIFICATION_TYPE_CHOICES, ) - notification_configuration = prevent_search(models.JSONField(default=dict)) + notification_configuration = prevent_search(JSONBlob(default=dict)) def default_messages(): return {'started': None, 'success': None, 'error': None, 'workflow_approval': None} - messages = models.JSONField(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.')) + messages = JSONBlob(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.')) def has_message(self, condition): potential_template = self.messages.get(condition, {}) @@ -236,7 +237,7 @@ class Notification(CreatedModifiedModel): default='', editable=False, ) - body = models.JSONField(default=dict, null=True, blank=True) + body = JSONBlob(default=dict, blank=True) def get_absolute_url(self, request=None): return reverse('api:notification_detail', kwargs={'pk': self.pk}, request=request) diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 385674d7ab..43f7fcb9ca 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -33,7 +33,7 @@ from awx.main.models.mixins import ResourceMixin, TaskManagerProjectUpdateMixin, from awx.main.utils import update_scm_url, polymorphic from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook from awx.main.utils.execution_environments import get_control_plane_execution_environment -from awx.main.fields import ImplicitRoleField +from awx.main.fields import ImplicitRoleField, JSONBlob from awx.main.models.rbac import ( ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR, @@ -293,7 +293,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn help_text=_('The last revision fetched by a project update'), ) - playbook_files = models.JSONField( + playbook_files = JSONBlob( default=list, blank=True, editable=False, @@ -301,7 +301,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn help_text=_('List of playbooks found in the project'), ) - inventory_files = models.JSONField( + inventory_files = JSONBlob( default=list, blank=True, editable=False, diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 0202360d66..63a5e2588a 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -54,7 +54,7 @@ from awx.main.utils import polymorphic from awx.main.constants import ACTIVE_STATES, CAN_CANCEL, JOB_VARIABLE_PREFIXES from awx.main.redact import UriCleaner, REPLACE_STR from awx.main.consumers import emit_channel_notification -from awx.main.fields import AskForField, OrderedManyToManyField +from awx.main.fields import AskForField, OrderedManyToManyField, JSONBlob __all__ = ['UnifiedJobTemplate', 'UnifiedJob', 'StdoutMaxBytesExceeded'] @@ -653,9 +653,8 @@ class UnifiedJob( editable=False, ) job_env = prevent_search( - models.JSONField( + JSONBlob( default=dict, - null=True, blank=True, editable=False, ) diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 197951ea05..30a2574748 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -28,7 +28,7 @@ from awx.main.models import prevent_search, accepts_json, UnifiedJobTemplate, Un from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin from awx.main.models.base import CreatedModifiedModel, VarsDictProperty from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR -from awx.main.fields import ImplicitRoleField, AskForField +from awx.main.fields import ImplicitRoleField, AskForField, JSONBlob from awx.main.models.mixins import ( ResourceMixin, SurveyJobTemplateMixin, @@ -231,9 +231,8 @@ class WorkflowJobNode(WorkflowNodeBase): default=None, on_delete=models.CASCADE, ) - ancestor_artifacts = models.JSONField( + ancestor_artifacts = JSONBlob( default=dict, - null=True, blank=True, editable=False, ) diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index 4397a79bed..7ee3849b36 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -1,6 +1,5 @@ # Python from collections import namedtuple -import itertools import functools import importlib import json @@ -14,7 +13,7 @@ from distutils.version import LooseVersion as Version # Django from django.conf import settings -from django.db import connection, transaction, DatabaseError, IntegrityError +from django.db import transaction, DatabaseError, IntegrityError from django.db.models.fields.related import ForeignKey from django.utils.timezone import now from django.utils.encoding import smart_str @@ -23,7 +22,6 @@ from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_noop from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist -from django.contrib.contenttypes.models import ContentType # Django-CRUM from crum import impersonate @@ -48,7 +46,6 @@ from awx.main.models import ( Inventory, SmartInventoryMembership, Job, - convert_jsonfields_to_jsonb, ) from awx.main.constants import ACTIVE_STATES from awx.main.dispatch.publish import task @@ -82,8 +79,6 @@ Try upgrading OpenSSH or providing your private key in an different format. \ def dispatch_startup(): startup_logger = logging.getLogger('awx.main.tasks') - convert_jsonfields_to_jsonb() - startup_logger.debug("Syncing Schedules") for sch in Schedule.objects.all(): try: @@ -127,123 +122,6 @@ def inform_cluster_of_shutdown(): logger.exception('Encountered problem with normal shutdown signal.') -def migrate_json_fields_expensive(table, columns): - batchsize = 50000 - - ct = ContentType.objects.get_by_natural_key(*table.split('_', 1)) - model = ct.model_class() - - # Phase 1: add the new columns, making them nullable to avoid populating them - with connection.schema_editor() as schema_editor: - # See: https://docs.djangoproject.com/en/3.1/ref/schema-editor/ - - for colname in columns: - f = model._meta.get_field(colname) - _, _, args, kwargs = f.deconstruct() - kwargs['null'] = True - new_f = f.__class__(*args, **kwargs) - new_f.set_attributes_from_name(f'_{colname}') - - schema_editor.add_field(model, new_f) - - # Create a trigger to make sure new data automatically gets put in both fields. - with connection.cursor() as cursor: - # It's a little annoying, I think this trigger will re-do - # the same work as the update query in Phase 2 - cursor.execute( - f""" - create or replace function update_{table}_{colname}() - returns trigger as $body$ - begin - new._{colname} = new.{colname}::jsonb - return new; - end - $body$ language plpgsql; - """ - ) - cursor.execute( - f""" - create trigger {table}_{colname}_trigger - before insert or update - on {table} - for each row - execute procedure update_{table}_{colname}; - """ - ) - - # Phase 2: copy over the data - with connection.cursor() as cursor: - rows = 0 - for i in itertools.count(0, batchsize): - cursor.execute(f"select count(1) from {table} where id >= %s;", (i,)) - if not cursor.fetchone()[0]: - break - - column_expr = ', '.join(f"_{colname} = {colname}::jsonb" for colname in columns) - cursor.execute( - f""" - update {table} - set {column_expr} - where id >= %s and id < %s; - """, - (i, i + batchsize), - ) - rows += cursor.rowcount - logger.debug(f"Batch {i} to {i + batchsize} copied on {table}.") - - logger.warning(f"Data copied for {rows} rows on {table}.") - - # Phase 3: drop the old column and rename the new one - with connection.schema_editor() as schema_editor: - - # FIXME: Grab a lock explicitly here? - for colname in columns: - with connection.cursor() as cursor: - cursor.execute(f"drop trigger {table}_{colname}_trigger;") - cursor.execute(f"drop function update_{table}_{colname};") - - f = model._meta.get_field(colname) - _, _, args, kwargs = f.deconstruct() - kwargs['null'] = True - new_f = f.__class__(*args, **kwargs) - new_f.set_attributes_from_name(f'_{colname}') - - schema_editor.remove_field(model, f) - - _, _, args, kwargs = new_f.deconstruct() - f = new_f.__class__(*args, **kwargs) - f.set_attributes_from_name(colname) - - schema_editor.alter_field(model, new_f, f) - - -@task(queue=get_local_queuename) -def migrate_json_fields(table, expensive, columns): - logger.warning(f"Migrating json fields: {table} {columns}") - - with advisory_lock(f'json_migration_{table}', wait=False) as acquired: - if not acquired: - return - - from django.db.migrations.executor import MigrationExecutor - - # If Django is currently running migrations, wait until it is done. - while True: - executor = MigrationExecutor(connection) - if not executor.migration_plan(executor.loader.graph.leaf_nodes()): - break - time.sleep(60) - - if expensive: - migrate_json_fields_expensive(table, columns) - else: - with connection.cursor() as cursor: - column_expr = " ".join(f"ALTER {colname} TYPE jsonb" for colname in columns) - cursor.execute(f"ALTER TABLE {table} {column_expr};") - - logger.warning(f"Migration of {table} to jsonb is finished") - - @task(queue=get_local_queuename) def apply_cluster_membership_policies(): from awx.main.signals import disable_activity_stream