External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-16 12:20:45 -03:30
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11947 from jbradberry/django-3.2-upgrade

Browse Source 
Remove the out-of-band JSONField migration
This commit is contained in:
Jeff Bradberry 2022-03-28 12:02:53 -04:00 committed by GitHub
commit 6c1adade25
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 102 additions and 288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
awx/conf/migrations/0001_initial.py
View File

@ -4,6 +4,8 @@ from __future__ import unicode_literals
from django.db import migrations, models
from django.conf import settings
import awx.main.fields
class Migration(migrations.Migration):
@ -17,7 +19,7 @@ class Migration(migrations.Migration):
('created', models.DateTimeField(default=None, editable=False)),
('modified', models.DateTimeField(default=None, editable=False)),
('key', models.CharField(max_length=255)),
('value', models.JSONField(null=True)),
('value', awx.main.fields.JSONBlob(null=True)),
(
'user',
models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True),

6
awx/conf/migrations/0003_v310_JSONField_changes.py
View File

@ -1,11 +1,13 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
from django.db import migrations
import awx.main.fields
class Migration(migrations.Migration):
dependencies = [('conf', '0002_v310_copy_tower_settings')]
operations = [migrations.AlterField(model_name='setting', name='value', field=models.JSONField(null=True))]
operations = [migrations.AlterField(model_name='setting', name='value', field=awx.main.fields.JSONBlob(null=True))]

3
awx/conf/models.py
View File

@ -8,6 +8,7 @@ import json
from django.db import models
# AWX
from awx.main.fields import JSONBlob
from awx.main.models.base import CreatedModifiedModel, prevent_search
from awx.main.utils import encrypt_field
from awx.conf import settings_registry
@ -18,7 +19,7 @@ __all__ = ['Setting']
class Setting(CreatedModifiedModel):
key = models.CharField(max_length=255)
value = models.JSONField(null=True)
value = JSONBlob(null=True)
user = prevent_search(models.ForeignKey('auth.User', related_name='settings', default=None, null=True, editable=False, on_delete=models.CASCADE))
def __str__(self):

11
awx/main/fields.py
View File

@ -43,7 +43,6 @@ from rest_framework import serializers
from awx.main.utils.filters import SmartFilter
from awx.main.utils.encryption import encrypt_value, decrypt_value, get_encryption_key
from awx.main.validators import validate_ssh_private_key
from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role, ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR
from awx.main.constants import ENV_BLOCKLIST
from awx.main import utils
@ -130,6 +129,9 @@ def is_implicit_parent(parent_role, child_role):
the model definition. This does not include any role parents that
might have been set by the user.
"""
# Avoid circular import
from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR
if child_role.content_object is None:
# The only singleton implicit parent is the system admin being
# a parent of the system auditor role
@ -286,6 +288,9 @@ class ImplicitRoleField(models.ForeignKey):
Model = utils.get_current_apps().get_model('main', instance.__class__.__name__)
latest_instance = Model.objects.get(pk=instance.pk)
# Avoid circular import
from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role
with batch_role_ancestor_rebuilding():
# Create any missing role objects
missing_roles = []
@ -340,6 +345,10 @@ class ImplicitRoleField(models.ForeignKey):
Role_ = utils.get_current_apps().get_model('main', 'Role')
child_ids = [x for x in Role_.parents.through.objects.filter(to_role_id__in=role_ids).distinct().values_list('from_role_id', flat=True)]
Role_.objects.filter(id__in=role_ids).delete()
# Avoid circular import
from awx.main.models.rbac import Role
Role.rebuild_role_ancestor_list([], child_ids)

6
awx/main/migrations/0001_initial.py
View File

@ -622,7 +622,7 @@ class Migration(migrations.Migration):
('dtend', models.DateTimeField(default=None, null=True, editable=False)),
('rrule', models.CharField(max_length=255)),
('next_run', models.DateTimeField(default=None, null=True, editable=False)),
('extra_data', models.JSONField(default=dict, null=True, blank=True)),
('extra_data', awx.main.fields.JSONBlob(default=dict, blank=True)),
(
'created_by',
models.ForeignKey(
@ -750,7 +750,7 @@ class Migration(migrations.Migration):
('elapsed', models.DecimalField(editable=False, max_digits=12, decimal_places=3)),
('job_args', models.TextField(default='', editable=False, blank=True)),
('job_cwd', models.CharField(default='', max_length=1024, editable=False, blank=True)),
('job_env', models.JSONField(default=dict, editable=False, null=True, blank=True)),
('job_env', awx.main.fields.JSONBlob(default=dict, editable=False, blank=True)),
('job_explanation', models.TextField(default='', editable=False, blank=True)),
('start_args', models.TextField(default='', editable=False, blank=True)),
('result_stdout_text', models.TextField(default='', editable=False, blank=True)),
@ -1034,7 +1034,7 @@ class Migration(migrations.Migration):
('host_config_key', models.CharField(default='', max_length=1024, blank=True)),
('ask_variables_on_launch', models.BooleanField(default=False)),
('survey_enabled', models.BooleanField(default=False)),
('survey_spec', models.JSONField(default=dict, blank=True)),
('survey_spec', awx.main.fields.JSONBlob(default=dict, blank=True)),
],
options={
'ordering': ('name',),

4
awx/main/migrations/0002_squashed_v300_release.py
View File

@ -198,7 +198,7 @@ class Migration(migrations.Migration):
),
('recipients', models.TextField(default='', editable=False, blank=True)),
('subject', models.TextField(default='', editable=False, blank=True)),
('body', models.JSONField(default=dict, null=True, blank=True)),
('body', awx.main.fields.JSONBlob(default=dict, blank=True)),
],
options={
'ordering': ('pk',),
@ -229,7 +229,7 @@ class Migration(migrations.Migration):
],
),
),
('notification_configuration', models.JSONField(default=dict)),
('notification_configuration', awx.main.fields.JSONBlob(default=dict)),
(
'created_by',
models.ForeignKey(

44
awx/main/migrations/0004_squashed_v310_release.py
View File

@ -220,7 +220,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobnode',
name='char_prompts',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AddField(
model_name='workflowjobnode',
@ -259,7 +259,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobtemplatenode',
name='char_prompts',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AddField(
model_name='workflowjobtemplatenode',
@ -307,12 +307,12 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='job',
name='artifacts',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
migrations.AddField(
model_name='workflowjobnode',
name='ancestor_artifacts',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
# Job timeout settings
migrations.AddField(
@ -380,7 +380,9 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='project',
name='playbook_files',
field=models.JSONField(default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True),
field=awx.main.fields.JSONBlob(
default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True
),
),
# Job events to stdout
migrations.AddField(
@ -536,7 +538,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjob',
name='survey_passwords',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
migrations.AddField(
model_name='workflowjobtemplate',
@ -546,7 +548,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobtemplate',
name='survey_spec',
field=models.JSONField(default=dict, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
# JSON field changes
migrations.AlterField(
@ -557,12 +559,12 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='job',
name='artifacts',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
migrations.AlterField(
model_name='job',
name='survey_passwords',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
migrations.AlterField(
model_name='jobevent',
@ -572,57 +574,59 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='jobtemplate',
name='survey_spec',
field=models.JSONField(default=dict, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AlterField(
model_name='notification',
name='body',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AlterField(
model_name='notificationtemplate',
name='notification_configuration',
field=models.JSONField(default=dict),
field=awx.main.fields.JSONBlob(default=dict),
),
migrations.AlterField(
model_name='project',
name='playbook_files',
field=models.JSONField(default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True),
field=awx.main.fields.JSONBlob(
default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True
),
),
migrations.AlterField(
model_name='schedule',
name='extra_data',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AlterField(
model_name='unifiedjob',
name='job_env',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
migrations.AlterField(
model_name='workflowjob',
name='survey_passwords',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
migrations.AlterField(
model_name='workflowjobnode',
name='ancestor_artifacts',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
migrations.AlterField(
model_name='workflowjobnode',
name='char_prompts',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AlterField(
model_name='workflowjobtemplate',
name='survey_spec',
field=models.JSONField(default=dict, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AlterField(
model_name='workflowjobtemplatenode',
name='char_prompts',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
# Job Project Update
migrations.AddField(

2
awx/main/migrations/0006_v320_release.py
View File

@ -175,7 +175,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='project',
name='inventory_files',
field=models.JSONField(
field=awx.main.fields.JSONBlob(
default=list,
help_text='Suggested list of content that could be Ansible inventory in the project',
verbose_name='Inventory Files',

6
awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py
View File

@ -1,7 +1,9 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
from django.db import migrations
import awx.main.fields
class Migration(migrations.Migration):
@ -14,6 +16,6 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='activitystream',
name='setting',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
]

18
awx/main/migrations/0014_v330_saved_launchtime_configs.py
View File

@ -20,7 +20,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='schedule',
name='char_prompts',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AddField(
model_name='schedule',
@ -37,7 +37,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='schedule',
name='survey_passwords',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
migrations.AddField(
model_name='workflowjobnode',
@ -47,12 +47,12 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobnode',
name='extra_data',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AddField(
model_name='workflowjobnode',
name='survey_passwords',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
migrations.AddField(
model_name='workflowjobtemplatenode',
@ -62,12 +62,12 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobtemplatenode',
name='extra_data',
field=models.JSONField(default=dict, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AddField(
model_name='workflowjobtemplatenode',
name='survey_passwords',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
# Run data migration before removing the old credential field
migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrations.RunPython.noop),
@ -85,9 +85,9 @@ class Migration(migrations.Migration):
name='JobLaunchConfig',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('extra_data', models.JSONField(blank=True, null=True, default=dict)),
('survey_passwords', models.JSONField(blank=True, null=True, default=dict, editable=False)),
('char_prompts', models.JSONField(blank=True, null=True, default=dict)),
('extra_data', awx.main.fields.JSONBlob(blank=True, default=dict)),
('survey_passwords', awx.main.fields.JSONBlob(blank=True, default=dict, editable=False)),
('char_prompts', awx.main.fields.JSONBlob(blank=True, default=dict)),
('credentials', models.ManyToManyField(related_name='joblaunchconfigs', to='main.Credential')),
(
'inventory',

6
awx/main/migrations/0020_v330_instancegroup_policies.py
View File

@ -1,8 +1,10 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from decimal import Decimal
from django.db import migrations, models
from decimal import Decimal
import awx.main.fields
class Migration(migrations.Migration):
@ -15,7 +17,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='instancegroup',
name='policy_instance_list',
field=models.JSONField(
field=awx.main.fields.JSONBlob(
default=list, help_text='List of exact-match Instances that will always be automatically assigned to this group', blank=True
),
),

6
awx/main/migrations/0038_v330_add_deleted_activitystream_actor.py
View File

@ -2,7 +2,9 @@
# Generated by Django 1.11.11 on 2018-05-21 19:51
from __future__ import unicode_literals
from django.db import models, migrations
from django.db import migrations
import awx.main.fields
class Migration(migrations.Migration):
@ -15,6 +17,6 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='activitystream',
name='deleted_actor',
field=models.JSONField(null=True),
field=awx.main.fields.JSONBlob(null=True),
),
]

5
awx/main/migrations/0053_v340_workflow_inventory.py
View File

@ -2,10 +2,11 @@
# Generated by Django 1.11.11 on 2018-09-27 19:50
from __future__ import unicode_literals
import awx.main.fields
from django.db import migrations, models
import django.db.models.deletion
import awx.main.fields
class Migration(migrations.Migration):
@ -17,7 +18,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjob',
name='char_prompts',
field=models.JSONField(blank=True, null=True, default=dict),
field=awx.main.fields.JSONBlob(blank=True, default=dict),
),
migrations.AddField(
model_name='workflowjob',

3
awx/main/migrations/0085_v360_add_notificationtemplate_messages.py
View File

@ -4,6 +4,7 @@ from __future__ import unicode_literals
from django.db import migrations, models
import awx.main.fields
import awx.main.models.notifications
@ -17,7 +18,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='notificationtemplate',
name='messages',
field=models.JSONField(
field=awx.main.fields.JSONBlob(
default=awx.main.models.notifications.NotificationTemplate.default_messages,
help_text='Optional custom messages for notification template.',
null=True,

2
awx/main/migrations/0090_v360_WFJT_prompts.py
View File

@ -24,7 +24,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobtemplate',
name='char_prompts',
field=models.JSONField(blank=True, null=True, default=dict),
field=awx.main.fields.JSONBlob(blank=True, default=dict),
),
migrations.AlterField(
model_name='joblaunchconfig',

2
awx/main/migrations/_squashed_30.py
View File

@ -29,7 +29,7 @@ SQUASHED_30 = {
migrations.AddField(
model_name='job',
name='survey_passwords',
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
field=awx.main.fields.JSONBlob(default=dict, editable=False, blank=True),
),
],
'0031_v302_migrate_survey_passwords': [

88
awx/main/models/__init__.py
View File

@ -3,7 +3,6 @@
# Django
from django.conf import settings # noqa
from django.db import connection
from django.db.models.signals import pre_delete # noqa
# AWX
@ -98,93 +97,6 @@ User.add_to_class('can_access_with_errors', check_user_access_with_errors)
User.add_to_class('accessible_objects', user_accessible_objects)
def convert_jsonfields_to_jsonb():
if connection.vendor != 'postgresql':
return
# fmt: off
fields = [ # Table name, expensive or not, tuple of column names
('conf_setting', False, (
'value',
)),
('main_instancegroup', False, (
'policy_instance_list',
)),
('main_jobtemplate', False, (
'survey_spec',
)),
('main_notificationtemplate', False, (
'notification_configuration',
'messages',
)),
('main_project', False, (
'playbook_files',
'inventory_files',
)),
('main_schedule', False, (
'extra_data',
'char_prompts',
'survey_passwords',
)),
('main_workflowjobtemplate', False, (
'survey_spec',
'char_prompts',
)),
('main_workflowjobtemplatenode', False, (
'char_prompts',
'extra_data',
'survey_passwords',
)),
('main_activitystream', True, (
'setting', # NN = NOT NULL
'deleted_actor',
)),
('main_job', True, (
'survey_passwords', # NN
'artifacts', # NN
)),
('main_joblaunchconfig', True, (
'extra_data', # NN
'survey_passwords', # NN
'char_prompts', # NN
)),
('main_notification', True, (
'body', # NN
)),
('main_unifiedjob', True, (
'job_env', # NN
)),
('main_workflowjob', True, (
'survey_passwords', # NN
'char_prompts', # NN
)),
('main_workflowjobnode', True, (
'char_prompts', # NN
'ancestor_artifacts', # NN
'extra_data', # NN
'survey_passwords', # NN
)),
]
# fmt: on
with connection.cursor() as cursor:
for table, expensive, columns in fields:
cursor.execute(
"""
select count(1) from information_schema.columns
where
table_name = %s and
column_name in %s and
data_type != 'jsonb';
""",
(table, columns),
)
if cursor.fetchone()[0]:
from awx.main.tasks.system import migrate_json_fields
migrate_json_fields.apply_async([table, expensive, columns])
def cleanup_created_modified_by(sender, **kwargs):
# work around a bug in django-polymorphic that doesn't properly
# handle cascades for reverse foreign keys on the polymorphic base model

5
awx/main/models/activity_stream.py
View File

@ -3,6 +3,7 @@
# AWX
from awx.api.versioning import reverse
from awx.main.fields import JSONBlob
from awx.main.models.base import accepts_json
# Django
@ -35,7 +36,7 @@ class ActivityStream(models.Model):
operation = models.CharField(max_length=13, choices=OPERATION_CHOICES)
timestamp = models.DateTimeField(auto_now_add=True)
changes = accepts_json(models.TextField(blank=True))
deleted_actor = models.JSONField(null=True)
deleted_actor = JSONBlob(null=True)
action_node = models.CharField(
blank=True,
default='',
@ -83,7 +84,7 @@ class ActivityStream(models.Model):
o_auth2_application = models.ManyToManyField("OAuth2Application", blank=True)
o_auth2_access_token = models.ManyToManyField("OAuth2AccessToken", blank=True)
setting = models.JSONField(default=dict, null=True, blank=True)
setting = JSONBlob(default=dict, blank=True)
def __str__(self):
operation = self.operation if 'operation' in self.__dict__ else '_delayed_'

3
awx/main/models/ha.py
View File

@ -19,6 +19,7 @@ from solo.models import SingletonModel
from awx import __version__ as awx_application_version
from awx.api.versioning import reverse
from awx.main.fields import JSONBlob
from awx.main.managers import InstanceManager, InstanceGroupManager, UUID_DEFAULT
from awx.main.constants import JOB_FOLDER_PREFIX
from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
@ -328,7 +329,7 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
)
policy_instance_percentage = models.IntegerField(default=0, help_text=_("Percentage of Instances to automatically assign to this group"))
policy_instance_minimum = models.IntegerField(default=0, help_text=_("Static minimum number of Instances to automatically assign to this group"))
policy_instance_list = models.JSONField(
policy_instance_list = JSONBlob(
default=list, blank=True, help_text=_("List of exact-match Instances that will always be automatically assigned to this group")
)

12
awx/main/models/jobs.py
View File

@ -44,7 +44,7 @@ from awx.main.models.notifications import (
JobNotificationMixin,
)
from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField
from awx.main.fields import ImplicitRoleField, AskForField
from awx.main.fields import ImplicitRoleField, AskForField, JSONBlob
from awx.main.models.mixins import (
ResourceMixin,
SurveyJobTemplateMixin,
@ -547,9 +547,8 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
editable=False,
through='JobHostSummary',
)
artifacts = models.JSONField(
artifacts = JSONBlob(
default=dict,
null=True,
blank=True,
editable=False,
)
@ -887,7 +886,7 @@ class LaunchTimeConfigBase(BaseModel):
)
# All standard fields are stored in this dictionary field
# This is a solution to the nullable CharField problem, specific to prompting
char_prompts = models.JSONField(default=dict, null=True, blank=True)
char_prompts = JSONBlob(default=dict, blank=True)
def prompts_dict(self, display=False):
data = {}
@ -940,12 +939,11 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
abstract = True
# Special case prompting fields, even more special than the other ones
extra_data = models.JSONField(default=dict, null=True, blank=True)
extra_data = JSONBlob(default=dict, blank=True)
survey_passwords = prevent_search(
models.JSONField(
JSONBlob(
default=dict,
editable=False,
null=True,
blank=True,
)
)

7
awx/main/models/mixins.py
View File

@ -24,7 +24,7 @@ from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_lice
from awx.main.utils.execution_environments import get_default_execution_environment
from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
from awx.main.fields import AskForField
from awx.main.fields import AskForField, JSONBlob
from awx.main.constants import ACTIVE_STATES
@ -103,7 +103,7 @@ class SurveyJobTemplateMixin(models.Model):
survey_enabled = models.BooleanField(
default=False,
)
survey_spec = prevent_search(models.JSONField(default=dict, blank=True))
survey_spec = prevent_search(JSONBlob(default=dict, blank=True))
ask_variables_on_launch = AskForField(blank=True, default=False, allows_field='extra_vars')
def survey_password_variables(self):
@ -365,10 +365,9 @@ class SurveyJobMixin(models.Model):
abstract = True
survey_passwords = prevent_search(
models.JSONField(
JSONBlob(
default=dict,
editable=False,
null=True,
blank=True,
)
)

7
awx/main/models/notifications.py
View File

@ -17,6 +17,7 @@ from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
# AWX
from awx.api.versioning import reverse
from awx.main.fields import JSONBlob
from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search
from awx.main.utils import encrypt_field, decrypt_field, set_environ
from awx.main.notifications.email_backend import CustomEmailBackend
@ -69,12 +70,12 @@ class NotificationTemplate(CommonModelNameNotUnique):
choices=NOTIFICATION_TYPE_CHOICES,
)
notification_configuration = prevent_search(models.JSONField(default=dict))
notification_configuration = prevent_search(JSONBlob(default=dict))
def default_messages():
return {'started': None, 'success': None, 'error': None, 'workflow_approval': None}
messages = models.JSONField(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.'))
messages = JSONBlob(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.'))
def has_message(self, condition):
potential_template = self.messages.get(condition, {})
@ -236,7 +237,7 @@ class Notification(CreatedModifiedModel):
default='',
editable=False,
)
body = models.JSONField(default=dict, null=True, blank=True)
body = JSONBlob(default=dict, blank=True)
def get_absolute_url(self, request=None):
return reverse('api:notification_detail', kwargs={'pk': self.pk}, request=request)

6
awx/main/models/projects.py
View File

@ -33,7 +33,7 @@ from awx.main.models.mixins import ResourceMixin, TaskManagerProjectUpdateMixin,
from awx.main.utils import update_scm_url, polymorphic
from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook
from awx.main.utils.execution_environments import get_control_plane_execution_environment
from awx.main.fields import ImplicitRoleField
from awx.main.fields import ImplicitRoleField, JSONBlob
from awx.main.models.rbac import (
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
ROLE_SINGLETON_SYSTEM_AUDITOR,
@ -293,7 +293,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
help_text=_('The last revision fetched by a project update'),
)
playbook_files = models.JSONField(
playbook_files = JSONBlob(
default=list,
blank=True,
editable=False,
@ -301,7 +301,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
help_text=_('List of playbooks found in the project'),
)
inventory_files = models.JSONField(
inventory_files = JSONBlob(
default=list,
blank=True,
editable=False,

5
awx/main/models/unified_jobs.py
View File

@ -54,7 +54,7 @@ from awx.main.utils import polymorphic
from awx.main.constants import ACTIVE_STATES, CAN_CANCEL, JOB_VARIABLE_PREFIXES
from awx.main.redact import UriCleaner, REPLACE_STR
from awx.main.consumers import emit_channel_notification
from awx.main.fields import AskForField, OrderedManyToManyField
from awx.main.fields import AskForField, OrderedManyToManyField, JSONBlob
__all__ = ['UnifiedJobTemplate', 'UnifiedJob', 'StdoutMaxBytesExceeded']
@ -653,9 +653,8 @@ class UnifiedJob(
editable=False,
)
job_env = prevent_search(
models.JSONField(
JSONBlob(
default=dict,
null=True,
blank=True,
editable=False,
)

5
awx/main/models/workflow.py
View File

@ -28,7 +28,7 @@ from awx.main.models import prevent_search, accepts_json, UnifiedJobTemplate, Un
from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin
from awx.main.models.base import CreatedModifiedModel, VarsDictProperty
from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR
from awx.main.fields import ImplicitRoleField, AskForField
from awx.main.fields import ImplicitRoleField, AskForField, JSONBlob
from awx.main.models.mixins import (
ResourceMixin,
SurveyJobTemplateMixin,
@ -231,9 +231,8 @@ class WorkflowJobNode(WorkflowNodeBase):
default=None,
on_delete=models.CASCADE,
)
ancestor_artifacts = models.JSONField(
ancestor_artifacts = JSONBlob(
default=dict,
null=True,
blank=True,
editable=False,
)

124
awx/main/tasks/system.py
View File

@ -1,6 +1,5 @@
# Python
from collections import namedtuple
import itertools
import functools
import importlib
import json
@ -14,7 +13,7 @@ from distutils.version import LooseVersion as Version
# Django
from django.conf import settings
from django.db import connection, transaction, DatabaseError, IntegrityError
from django.db import transaction, DatabaseError, IntegrityError
from django.db.models.fields.related import ForeignKey
from django.utils.timezone import now
from django.utils.encoding import smart_str
@ -23,7 +22,6 @@ from django.utils.translation import gettext_lazy as _
from django.utils.translation import gettext_noop
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.contenttypes.models import ContentType
# Django-CRUM
from crum import impersonate
@ -48,7 +46,6 @@ from awx.main.models import (
Inventory,
SmartInventoryMembership,
Job,
convert_jsonfields_to_jsonb,
)
from awx.main.constants import ACTIVE_STATES
from awx.main.dispatch.publish import task
@ -82,8 +79,6 @@ Try upgrading OpenSSH or providing your private key in an different format. \
def dispatch_startup():
startup_logger = logging.getLogger('awx.main.tasks')
convert_jsonfields_to_jsonb()
startup_logger.debug("Syncing Schedules")
for sch in Schedule.objects.all():
try:
@ -127,123 +122,6 @@ def inform_cluster_of_shutdown():
logger.exception('Encountered problem with normal shutdown signal.')
def migrate_json_fields_expensive(table, columns):
batchsize = 50000
ct = ContentType.objects.get_by_natural_key(*table.split('_', 1))
model = ct.model_class()
# Phase 1: add the new columns, making them nullable to avoid populating them
with connection.schema_editor() as schema_editor:
# See: https://docs.djangoproject.com/en/3.1/ref/schema-editor/
for colname in columns:
f = model._meta.get_field(colname)
_, _, args, kwargs = f.deconstruct()
kwargs['null'] = True
new_f = f.__class__(*args, **kwargs)
new_f.set_attributes_from_name(f'_{colname}')
schema_editor.add_field(model, new_f)
# Create a trigger to make sure new data automatically gets put in both fields.
with connection.cursor() as cursor:
# It's a little annoying, I think this trigger will re-do
# the same work as the update query in Phase 2
cursor.execute(
f"""
create or replace function update_{table}_{colname}()
returns trigger as $body$
begin
new._{colname} = new.{colname}::jsonb
return new;
end
$body$ language plpgsql;
"""
)
cursor.execute(
f"""
create trigger {table}_{colname}_trigger
before insert or update
on {table}
for each row
execute procedure update_{table}_{colname};
"""
)
# Phase 2: copy over the data
with connection.cursor() as cursor:
rows = 0
for i in itertools.count(0, batchsize):
cursor.execute(f"select count(1) from {table} where id >= %s;", (i,))
if not cursor.fetchone()[0]:
break
column_expr = ', '.join(f"_{colname} = {colname}::jsonb" for colname in columns)
cursor.execute(
f"""
update {table}
set {column_expr}
where id >= %s and id < %s;
""",
(i, i + batchsize),
)
rows += cursor.rowcount
logger.debug(f"Batch {i} to {i + batchsize} copied on {table}.")
logger.warning(f"Data copied for {rows} rows on {table}.")
# Phase 3: drop the old column and rename the new one
with connection.schema_editor() as schema_editor:
# FIXME: Grab a lock explicitly here?
for colname in columns:
with connection.cursor() as cursor:
cursor.execute(f"drop trigger {table}_{colname}_trigger;")
cursor.execute(f"drop function update_{table}_{colname};")
f = model._meta.get_field(colname)
_, _, args, kwargs = f.deconstruct()
kwargs['null'] = True
new_f = f.__class__(*args, **kwargs)
new_f.set_attributes_from_name(f'_{colname}')
schema_editor.remove_field(model, f)
_, _, args, kwargs = new_f.deconstruct()
f = new_f.__class__(*args, **kwargs)
f.set_attributes_from_name(colname)
schema_editor.alter_field(model, new_f, f)
@task(queue=get_local_queuename)
def migrate_json_fields(table, expensive, columns):
logger.warning(f"Migrating json fields: {table} {columns}")
with advisory_lock(f'json_migration_{table}', wait=False) as acquired:
if not acquired:
return
from django.db.migrations.executor import MigrationExecutor
# If Django is currently running migrations, wait until it is done.
while True:
executor = MigrationExecutor(connection)
if not executor.migration_plan(executor.loader.graph.leaf_nodes()):
break
time.sleep(60)
if expensive:
migrate_json_fields_expensive(table, columns)
else:
with connection.cursor() as cursor:
column_expr = " ".join(f"ALTER {colname} TYPE jsonb" for colname in columns)
cursor.execute(f"ALTER TABLE {table} {column_expr};")
logger.warning(f"Migration of {table} to jsonb is finished")
@task(queue=get_local_queuename)
def apply_cluster_membership_policies():
from awx.main.signals import disable_activity_stream