External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-02-27 07:56:06 -03:30
Code Issues Packages Projects Releases Wiki Activity

Replace all usage of customized json fields with the Django builtin

Browse Source 
The event_data field on event models, however, is getting an
overridden version that retains the underlying text data type for the
column, to avoid a heavy data migration on those tables.

Also, certain of the larger tables are getting these fields with the
NOT NULL constraint turned off, to avoid a long migration.

Remove the django.utils.six monkey patch we did at the beginning of
the upgrade.
This commit is contained in:
Jeff Bradberry
2022-02-03 10:29:08 -05:00
parent 65d17fb316
commit 05142a779d
36 changed files with 122 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
awx/__init__.py
View File

@@ -6,8 +6,6 @@ import os
import sys
import warnings
import six
from pkg_resources import get_distribution
__version__ = get_distribution('awx').version
@@ -37,9 +35,6 @@ else:
from django.db.models import indexes
from django.db.backends.utils import names_digest
from django.db import connection
from django import utils
utils.six = six # FIXME: monkey patch to get us through for now
if HAS_DJANGO is True:

3
awx/api/metadata.py
View File

@@ -6,6 +6,7 @@ from uuid import UUID
# Django
from django.core.exceptions import PermissionDenied
from django.db.models import JSONField
from django.db.models.fields import PositiveIntegerField, BooleanField
from django.db.models.fields.related import ForeignKey
from django.http import Http404
@@ -22,7 +23,7 @@ from rest_framework.request import clone_request
# AWX
from awx.api.fields import ChoiceNullField
from awx.main.fields import JSONField, ImplicitRoleField
from awx.main.fields import ImplicitRoleField
from awx.main.models import NotificationTemplate
from awx.main.utils.execution_environments import get_default_pod_spec

4
awx/api/serializers.py
View File

@@ -97,7 +97,7 @@ from awx.main.models import (
)
from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES
from awx.main.models.rbac import get_roles_on_resource, role_summary_fields_generator
from awx.main.fields import ImplicitRoleField, JSONBField
from awx.main.fields import ImplicitRoleField
from awx.main.utils import (
get_type_for_model,
get_model_for_type,
@@ -1718,7 +1718,7 @@ class InventorySerializer(LabelsListMixin, BaseSerializerWithVariables):
def validate_host_filter(self, host_filter):
if host_filter:
try:
for match in JSONBField.get_lookups().keys():
for match in models.JSONField.get_lookups().keys():
if match == 'exact':
# __exact is allowed
continue

3
awx/conf/migrations/0001_initial.py
View File

@@ -2,7 +2,6 @@
from __future__ import unicode_literals
from django.db import migrations, models
import jsonfield.fields
from django.conf import settings
@@ -18,7 +17,7 @@ class Migration(migrations.Migration):
('created', models.DateTimeField(default=None, editable=False)),
('modified', models.DateTimeField(default=None, editable=False)),
('key', models.CharField(max_length=255)),
('value', jsonfield.fields.JSONField(null=True)),
('value', models.JSONField(null=True)),
(
'user',
models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True),

5
awx/conf/migrations/0003_v310_JSONField_changes.py
View File

@@ -1,12 +1,11 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations
import awx.main.fields
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [('conf', '0002_v310_copy_tower_settings')]
operations = [migrations.AlterField(model_name='setting', name='value', field=awx.main.fields.JSONField(null=True))]
operations = [migrations.AlterField(model_name='setting', name='value', field=models.JSONField(null=True))]

3
awx/conf/models.py
View File

@@ -9,7 +9,6 @@ from django.db import models
# AWX
from awx.main.models.base import CreatedModifiedModel, prevent_search
from awx.main.fields import JSONField
from awx.main.utils import encrypt_field
from awx.conf import settings_registry
@@ -19,7 +18,7 @@ __all__ = ['Setting']
class Setting(CreatedModifiedModel):
key = models.CharField(max_length=255)
value = JSONField(null=True)
value = models.JSONField(null=True)
user = prevent_search(models.ForeignKey('auth.User', related_name='settings', default=None, null=True, editable=False, on_delete=models.CASCADE))
def __str__(self):

53
awx/main/fields.py
View File

@@ -11,7 +11,6 @@ from jinja2 import sandbox, StrictUndefined
from jinja2.exceptions import UndefinedError, TemplateSyntaxError, SecurityError
# Django
from django.contrib.postgres.fields import JSONField as upstream_JSONBField
from django.core import exceptions as django_exceptions
from django.core.serializers.json import DjangoJSONEncoder
from django.db.models.signals import (
@@ -29,6 +28,7 @@ from django.db.models.fields.related_descriptors import (
create_forward_many_to_many_manager,
)
from django.utils.encoding import smart_str
from django.db.models import JSONField
from django.utils.functional import cached_property
from django.utils.translation import gettext_lazy as _
@@ -36,9 +36,6 @@ from django.utils.translation import gettext_lazy as _
from jsonschema import Draft4Validator, FormatChecker
import jsonschema.exceptions
# Django-JSONField
from jsonfield import JSONField as upstream_JSONField
# DRF
from rest_framework import serializers
@@ -52,9 +49,9 @@ from awx.main import utils
__all__ = [
'JSONBlob',
'AutoOneToOneField',
'ImplicitRoleField',
'JSONField',
'SmartFilterField',
'OrderedManyToManyField',
'update_role_parentage_for_instance',
@@ -71,40 +68,9 @@ def __enum_validate__(validator, enums, instance, schema):
Draft4Validator.VALIDATORS['enum'] = __enum_validate__
class JSONField(upstream_JSONField):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.decoder_kwargs = {'cls': json.JSONDecoder} # FIXME
def db_type(self, connection):
return 'text'
def from_db_value(self, value, expression, connection):
if value in {'', None} and not self.null:
return {}
return super(JSONField, self).from_db_value(value, expression, connection)
class JSONBField(upstream_JSONBField):
def get_prep_lookup(self, lookup_type, value):
if isinstance(value, str) and value == "null":
return 'null'
return super(JSONBField, self).get_prep_lookup(lookup_type, value)
def get_db_prep_value(self, value, connection, prepared=False):
if connection.vendor == 'sqlite':
# sqlite (which we use for tests) does not support jsonb;
if hasattr(value, 'adapted'):
value = value.adapted # FIXME: Django 3.0 uses JsonAdapter, removed in 3.1
return json.dumps(value, cls=DjangoJSONEncoder)
return super(JSONBField, self).get_db_prep_value(value, connection, prepared)
def from_db_value(self, value, expression, connection):
# Work around a bug in django-jsonfield
# https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos
if isinstance(value, str):
return json.loads(value)
return value
class JSONBlob(JSONField):
def get_internal_type(self):
return "TextField"
# Based on AutoOneToOneField from django-annoying:
@@ -391,7 +357,7 @@ class SmartFilterField(models.TextField):
return super(SmartFilterField, self).get_prep_value(value)
class JSONSchemaField(JSONBField):
class JSONSchemaField(models.JSONField):
"""
A JSONB field that self-validates against a defined JSON schema
(http://json-schema.org). This base class is intended to be overwritten by
@@ -404,8 +370,13 @@ class JSONSchemaField(JSONBField):
# validation
empty_values = (None, '')
def __init__(self, encoder=None, decoder=None, **options):
if encoder is None:
encoder = DjangoJSONEncoder
super().__init__(encoder=encoder, decoder=decoder, **options)
def get_default(self):
return copy.deepcopy(super(JSONBField, self).get_default())
return copy.deepcopy(super(models.JSONField, self).get_default())
def schema(self, model_instance):
raise NotImplementedError()

11
awx/main/migrations/0001_initial.py
View File

@@ -7,7 +7,6 @@ from __future__ import unicode_literals
from django.db import migrations, models
import django.utils.timezone
import jsonfield.fields
import django.db.models.deletion
from django.conf import settings
import taggit.managers
@@ -70,7 +69,7 @@ class Migration(migrations.Migration):
],
),
),
('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
('event_data', awx.main.fields.JSONBlob(default=dict, blank=True)),
('failed', models.BooleanField(default=False, editable=False)),
('changed', models.BooleanField(default=False, editable=False)),
('counter', models.PositiveIntegerField(default=0)),
@@ -433,7 +432,7 @@ class Migration(migrations.Migration):
],
),
),
('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
('event_data', awx.main.fields.JSONBlob(default=dict, blank=True)),
('failed', models.BooleanField(default=False, editable=False)),
('changed', models.BooleanField(default=False, editable=False)),
('host_name', models.CharField(default='', max_length=1024, editable=False)),
@@ -623,7 +622,7 @@ class Migration(migrations.Migration):
('dtend', models.DateTimeField(default=None, null=True, editable=False)),
('rrule', models.CharField(max_length=255)),
('next_run', models.DateTimeField(default=None, null=True, editable=False)),
('extra_data', jsonfield.fields.JSONField(default=dict, blank=True)),
('extra_data', models.JSONField(default=dict, null=True, blank=True)),
(
'created_by',
models.ForeignKey(
@@ -751,7 +750,7 @@ class Migration(migrations.Migration):
('elapsed', models.DecimalField(editable=False, max_digits=12, decimal_places=3)),
('job_args', models.TextField(default='', editable=False, blank=True)),
('job_cwd', models.CharField(default='', max_length=1024, editable=False, blank=True)),
('job_env', jsonfield.fields.JSONField(default=dict, editable=False, blank=True)),
('job_env', models.JSONField(default=dict, editable=False, null=True, blank=True)),
('job_explanation', models.TextField(default='', editable=False, blank=True)),
('start_args', models.TextField(default='', editable=False, blank=True)),
('result_stdout_text', models.TextField(default='', editable=False, blank=True)),
@@ -1035,7 +1034,7 @@ class Migration(migrations.Migration):
('host_config_key', models.CharField(default='', max_length=1024, blank=True)),
('ask_variables_on_launch', models.BooleanField(default=False)),
('survey_enabled', models.BooleanField(default=False)),
('survey_spec', jsonfield.fields.JSONField(default=dict, blank=True)),
('survey_spec', models.JSONField(default=dict, blank=True)),
],
options={
'ordering': ('name',),

9
awx/main/migrations/0002_squashed_v300_release.py
View File

@@ -12,7 +12,6 @@ import django.db.models.deletion
from django.conf import settings
from django.utils.timezone import now
import jsonfield.fields
import taggit.managers
@@ -199,7 +198,7 @@ class Migration(migrations.Migration):
),
('recipients', models.TextField(default='', editable=False, blank=True)),
('subject', models.TextField(default='', editable=False, blank=True)),
('body', jsonfield.fields.JSONField(default=dict, blank=True)),
('body', models.JSONField(default=dict, null=True, blank=True)),
],
options={
'ordering': ('pk',),
@@ -230,7 +229,7 @@ class Migration(migrations.Migration):
],
),
),
('notification_configuration', jsonfield.fields.JSONField(default=dict)),
('notification_configuration', models.JSONField(default=dict)),
(
'created_by',
models.ForeignKey(
@@ -324,9 +323,7 @@ class Migration(migrations.Migration):
('module', models.CharField(max_length=128)),
(
'facts',
awx.main.fields.JSONBField(
default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True
),
models.JSONField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
),
(
'host',

49
awx/main/migrations/0004_squashed_v310_release.py
View File

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
from django.db import migrations, models
import awx.main.models.notifications
import jsonfield.fields
import django.db.models.deletion
import awx.main.models.workflow
import awx.main.fields
@@ -221,7 +220,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobnode',
name='char_prompts',
field=jsonfield.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
migrations.AddField(
model_name='workflowjobnode',
@@ -260,7 +259,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobtemplatenode',
name='char_prompts',
field=jsonfield.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
migrations.AddField(
model_name='workflowjobtemplatenode',
@@ -308,12 +307,12 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='job',
name='artifacts',
field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
migrations.AddField(
model_name='workflowjobnode',
name='ancestor_artifacts',
field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
# Job timeout settings
migrations.AddField(
@@ -381,9 +380,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='project',
name='playbook_files',
field=jsonfield.fields.JSONField(
default=[], help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True
),
field=models.JSONField(default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True),
),
# Job events to stdout
migrations.AddField(
@@ -539,7 +536,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjob',
name='survey_passwords',
field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
migrations.AddField(
model_name='workflowjobtemplate',
@@ -549,85 +546,83 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobtemplate',
name='survey_spec',
field=jsonfield.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, blank=True),
),
# JSON field changes
migrations.AlterField(
model_name='adhoccommandevent',
name='event_data',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AlterField(
model_name='job',
name='artifacts',
field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
migrations.AlterField(
model_name='job',
name='survey_passwords',
field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
migrations.AlterField(
model_name='jobevent',
name='event_data',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=awx.main.fields.JSONBlob(default=dict, blank=True),
),
migrations.AlterField(
model_name='jobtemplate',
name='survey_spec',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, blank=True),
),
migrations.AlterField(
model_name='notification',
name='body',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
migrations.AlterField(
model_name='notificationtemplate',
name='notification_configuration',
field=awx.main.fields.JSONField(default=dict),
field=models.JSONField(default=dict),
),
migrations.AlterField(
model_name='project',
name='playbook_files',
field=awx.main.fields.JSONField(
default=[], help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True
),
field=models.JSONField(default=list, help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True),
),
migrations.AlterField(
model_name='schedule',
name='extra_data',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
migrations.AlterField(
model_name='unifiedjob',
name='job_env',
field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
migrations.AlterField(
model_name='workflowjob',
name='survey_passwords',
field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
migrations.AlterField(
model_name='workflowjobnode',
name='ancestor_artifacts',
field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
migrations.AlterField(
model_name='workflowjobnode',
name='char_prompts',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
migrations.AlterField(
model_name='workflowjobtemplate',
name='survey_spec',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, blank=True),
),
migrations.AlterField(
model_name='workflowjobtemplatenode',
name='char_prompts',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
# Job Project Update
migrations.AddField(

10
awx/main/migrations/0006_v320_release.py
View File

@@ -108,14 +108,12 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='fact',
name='facts',
field=awx.main.fields.JSONBField(
default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True
),
field=models.JSONField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
),
migrations.AddField(
model_name='host',
name='ansible_facts',
field=awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
field=models.JSONField(default=dict, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
),
migrations.AddField(
model_name='host',
@@ -177,8 +175,8 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='project',
name='inventory_files',
field=awx.main.fields.JSONField(
default=[],
field=models.JSONField(
default=list,
help_text='Suggested list of content that could be Ansible inventory in the project',
verbose_name='Inventory Files',
editable=False,

5
awx/main/migrations/0009_v322_add_setting_field_for_activity_stream.py
View File

@@ -1,8 +1,7 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations
import awx.main.fields
from django.db import migrations, models
class Migration(migrations.Migration):
@@ -15,6 +14,6 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='activitystream',
name='setting',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
]

18
awx/main/migrations/0014_v330_saved_launchtime_configs.py
View File

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='schedule',
name='char_prompts',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
migrations.AddField(
model_name='schedule',
@@ -37,7 +37,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='schedule',
name='survey_passwords',
field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
migrations.AddField(
model_name='workflowjobnode',
@@ -47,12 +47,12 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobnode',
name='extra_data',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
migrations.AddField(
model_name='workflowjobnode',
name='survey_passwords',
field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
migrations.AddField(
model_name='workflowjobtemplatenode',
@@ -62,12 +62,12 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobtemplatenode',
name='extra_data',
field=awx.main.fields.JSONField(default=dict, blank=True),
field=models.JSONField(default=dict, null=True, blank=True),
),
migrations.AddField(
model_name='workflowjobtemplatenode',
name='survey_passwords',
field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
# Run data migration before removing the old credential field
migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrations.RunPython.noop),
@@ -85,9 +85,9 @@ class Migration(migrations.Migration):
name='JobLaunchConfig',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('extra_data', awx.main.fields.JSONField(blank=True, default=dict)),
('survey_passwords', awx.main.fields.JSONField(blank=True, default=dict, editable=False)),
('char_prompts', awx.main.fields.JSONField(blank=True, default=dict)),
('extra_data', models.JSONField(blank=True, null=True, default=dict)),
('survey_passwords', models.JSONField(blank=True, null=True, default=dict, editable=False)),
('char_prompts', models.JSONField(blank=True, null=True, default=dict)),
('credentials', models.ManyToManyField(related_name='joblaunchconfigs', to='main.Credential')),
(
'inventory',

9
awx/main/migrations/0018_v330_add_additional_stdout_events.py
View File

@@ -2,10 +2,11 @@
# Generated by Django 1.11.7 on 2017-12-14 15:13
from __future__ import unicode_literals
import awx.main.fields
from django.db import migrations, models
import django.db.models.deletion
import awx.main.fields
class Migration(migrations.Migration):
@@ -20,7 +21,7 @@ class Migration(migrations.Migration):
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField(default=None, editable=False)),
('modified', models.DateTimeField(default=None, editable=False)),
('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
('event_data', awx.main.fields.JSONBlob(blank=True, default=dict)),
('uuid', models.CharField(default='', editable=False, max_length=1024)),
('counter', models.PositiveIntegerField(default=0, editable=False)),
('stdout', models.TextField(default='', editable=False)),
@@ -84,7 +85,7 @@ class Migration(migrations.Migration):
max_length=100,
),
),
('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
('event_data', awx.main.fields.JSONBlob(blank=True, default=dict)),
('failed', models.BooleanField(default=False, editable=False)),
('changed', models.BooleanField(default=False, editable=False)),
('uuid', models.CharField(default='', editable=False, max_length=1024)),
@@ -114,7 +115,7 @@ class Migration(migrations.Migration):
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField(default=None, editable=False)),
('modified', models.DateTimeField(default=None, editable=False)),
('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
('event_data', awx.main.fields.JSONBlob(blank=True, default=dict)),
('uuid', models.CharField(default='', editable=False, max_length=1024)),
('counter', models.PositiveIntegerField(default=0, editable=False)),
('stdout', models.TextField(default='', editable=False)),

5
awx/main/migrations/0020_v330_instancegroup_policies.py
View File

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
from django.db import migrations, models
from decimal import Decimal
import awx.main.fields
class Migration(migrations.Migration):
@@ -16,8 +15,8 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='instancegroup',
name='policy_instance_list',
field=awx.main.fields.JSONField(
default=[], help_text='List of exact-match Instances that will always be automatically assigned to this group', blank=True
field=models.JSONField(
default=list, help_text='List of exact-match Instances that will always be automatically assigned to this group', blank=True
),
),
migrations.AddField(

6
awx/main/migrations/0038_v330_add_deleted_activitystream_actor.py
View File

@@ -2,9 +2,7 @@
# Generated by Django 1.11.11 on 2018-05-21 19:51
from __future__ import unicode_literals
import awx.main.fields
import awx.main.models.activity_stream
from django.db import migrations
from django.db import models, migrations
class Migration(migrations.Migration):
@@ -17,6 +15,6 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='activitystream',
name='deleted_actor',
field=awx.main.fields.JSONField(null=True),
field=models.JSONField(null=True),
),
]

2
awx/main/migrations/0053_v340_workflow_inventory.py
View File

@@ -17,7 +17,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjob',
name='char_prompts',
field=awx.main.fields.JSONField(blank=True, default=dict),
field=models.JSONField(blank=True, null=True, default=dict),
),
migrations.AddField(
model_name='workflowjob',

3
awx/main/migrations/0085_v360_add_notificationtemplate_messages.py
View File

@@ -4,7 +4,6 @@ from __future__ import unicode_literals
from django.db import migrations, models
import awx.main.fields
import awx.main.models.notifications
@@ -18,7 +17,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='notificationtemplate',
name='messages',
field=awx.main.fields.JSONField(
field=models.JSONField(
default=awx.main.models.notifications.NotificationTemplate.default_messages,
help_text='Optional custom messages for notification template.',
null=True,

2
awx/main/migrations/0090_v360_WFJT_prompts.py
View File

@@ -24,7 +24,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='workflowjobtemplate',
name='char_prompts',
field=awx.main.fields.JSONField(blank=True, default=dict),
field=models.JSONField(blank=True, null=True, default=dict),
),
migrations.AlterField(
model_name='joblaunchconfig',

5
awx/main/migrations/0129_unifiedjob_installed_collections.py
View File

@@ -1,7 +1,6 @@
# Generated by Django 2.2.16 on 2021-02-16 20:27
import awx.main.fields
from django.db import migrations
from django.db import migrations, models
class Migration(migrations.Migration):
@@ -14,7 +13,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='unifiedjob',
name='installed_collections',
field=awx.main.fields.JSONBField(
field=models.JSONField(
blank=True, default=dict, editable=False, help_text='The Collections names and versions installed in the execution environment.'
),
),

3
awx/main/migrations/_squashed_30.py
View File

@@ -2,7 +2,6 @@ from django.db import (
migrations,
models,
)
import jsonfield.fields
import awx.main.fields
from awx.main.migrations import _save_password_keys
@@ -30,7 +29,7 @@ SQUASHED_30 = {
migrations.AddField(
model_name='job',
name='survey_passwords',
field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
field=models.JSONField(default=dict, editable=False, null=True, blank=True),
),
],
'0031_v302_migrate_survey_passwords': [

5
awx/main/models/activity_stream.py
View File

@@ -3,7 +3,6 @@
# AWX
from awx.api.versioning import reverse
from awx.main.fields import JSONField
from awx.main.models.base import accepts_json
# Django
@@ -36,7 +35,7 @@ class ActivityStream(models.Model):
operation = models.CharField(max_length=13, choices=OPERATION_CHOICES)
timestamp = models.DateTimeField(auto_now_add=True)
changes = accepts_json(models.TextField(blank=True))
deleted_actor = JSONField(null=True)
deleted_actor = models.JSONField(null=True)
action_node = models.CharField(
blank=True,
default='',
@@ -84,7 +83,7 @@ class ActivityStream(models.Model):
o_auth2_application = models.ManyToManyField("OAuth2Application", blank=True)
o_auth2_access_token = models.ManyToManyField("OAuth2AccessToken", blank=True)
setting = JSONField(blank=True)
setting = models.JSONField(default=dict, null=True, blank=True)
def __str__(self):
operation = self.operation if 'operation' in self.__dict__ else '_delayed_'

12
awx/main/models/events.py
View File

@@ -15,8 +15,8 @@ from django.utils.encoding import force_str
from awx.api.versioning import reverse
from awx.main import consumers
from awx.main.fields import JSONBlob
from awx.main.managers import DeferJobCreatedManager
from awx.main.fields import JSONField
from awx.main.constants import MINIMAL_EVENTS
from awx.main.models.base import CreatedModifiedModel
from awx.main.utils import ignore_inventory_computed_fields, camelcase_to_underscore
@@ -209,10 +209,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
max_length=100,
choices=EVENT_CHOICES,
)
event_data = JSONField(
blank=True,
default=dict,
)
event_data = JSONBlob(default=dict, blank=True)
failed = models.BooleanField(
default=False,
editable=False,
@@ -648,10 +645,7 @@ class BaseCommandEvent(CreatedModifiedModel):
class Meta:
abstract = True
event_data = JSONField(
blank=True,
default=dict,
)
event_data = JSONBlob(default=dict, blank=True)
uuid = models.CharField(
max_length=1024,
default='',

5
awx/main/models/ha.py
View File

@@ -19,7 +19,6 @@ from solo.models import SingletonModel
from awx import __version__ as awx_application_version
from awx.api.versioning import reverse
from awx.main.managers import InstanceManager, InstanceGroupManager, UUID_DEFAULT
from awx.main.fields import JSONField
from awx.main.constants import JOB_FOLDER_PREFIX
from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
from awx.main.models.unified_jobs import UnifiedJob
@@ -322,8 +321,8 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
)
policy_instance_percentage = models.IntegerField(default=0, help_text=_("Percentage of Instances to automatically assign to this group"))
policy_instance_minimum = models.IntegerField(default=0, help_text=_("Static minimum number of Instances to automatically assign to this group"))
policy_instance_list = JSONField(
default=[], blank=True, help_text=_("List of exact-match Instances that will always be automatically assigned to this group")
policy_instance_list = models.JSONField(
default=list, blank=True, help_text=_("List of exact-match Instances that will always be automatically assigned to this group")
)
POLICY_FIELDS = frozenset(('policy_instance_list', 'policy_instance_minimum', 'policy_instance_percentage'))

3
awx/main/models/inventory.py
View File

@@ -29,7 +29,6 @@ from awx.main.constants import CLOUD_PROVIDERS
from awx.main.consumers import emit_channel_notification
from awx.main.fields import (
ImplicitRoleField,
JSONBField,
SmartFilterField,
OrderedManyToManyField,
)
@@ -488,7 +487,7 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
editable=False,
help_text=_('Inventory source(s) that created or modified this host.'),
)
ansible_facts = JSONBField(
ansible_facts = models.JSONField(
blank=True,
default=dict,
help_text=_('Arbitrary JSON structure of most recent ansible_facts, per-host.'),

16
awx/main/models/jobs.py
View File

@@ -44,7 +44,7 @@ from awx.main.models.notifications import (
JobNotificationMixin,
)
from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField
from awx.main.fields import ImplicitRoleField, JSONField, AskForField
from awx.main.fields import ImplicitRoleField, AskForField
from awx.main.models.mixins import (
ResourceMixin,
SurveyJobTemplateMixin,
@@ -546,9 +546,10 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
editable=False,
through='JobHostSummary',
)
artifacts = JSONField(
blank=True,
artifacts = models.JSONField(
default=dict,
null=True,
blank=True,
editable=False,
)
scm_revision = models.CharField(
@@ -885,7 +886,7 @@ class LaunchTimeConfigBase(BaseModel):
)
# All standard fields are stored in this dictionary field
# This is a solution to the nullable CharField problem, specific to prompting
char_prompts = JSONField(blank=True, default=dict)
char_prompts = models.JSONField(default=dict, null=True, blank=True)
def prompts_dict(self, display=False):
data = {}
@@ -938,12 +939,13 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
abstract = True
# Special case prompting fields, even more special than the other ones
extra_data = JSONField(blank=True, default=dict)
extra_data = models.JSONField(default=dict, null=True, blank=True)
survey_passwords = prevent_search(
JSONField(
blank=True,
models.JSONField(
default=dict,
editable=False,
null=True,
blank=True,
)
)
# Credentials needed for non-unified job / unified JT models

14
awx/main/models/mixins.py
View File

@@ -24,7 +24,7 @@ from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_lice
from awx.main.utils.execution_environments import get_default_execution_environment
from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
from awx.main.fields import JSONField, AskForField
from awx.main.fields import AskForField
from awx.main.constants import ACTIVE_STATES
@@ -103,12 +103,7 @@ class SurveyJobTemplateMixin(models.Model):
survey_enabled = models.BooleanField(
default=False,
)
survey_spec = prevent_search(
JSONField(
blank=True,
default=dict,
)
)
survey_spec = prevent_search(models.JSONField(default=dict, blank=True))
ask_variables_on_launch = AskForField(blank=True, default=False, allows_field='extra_vars')
def survey_password_variables(self):
@@ -370,10 +365,11 @@ class SurveyJobMixin(models.Model):
abstract = True
survey_passwords = prevent_search(
JSONField(
blank=True,
models.JSONField(
default=dict,
editable=False,
null=True,
blank=True,
)
)

7
awx/main/models/notifications.py
View File

@@ -28,7 +28,6 @@ from awx.main.notifications.mattermost_backend import MattermostBackend
from awx.main.notifications.grafana_backend import GrafanaBackend
from awx.main.notifications.rocketchat_backend import RocketChatBackend
from awx.main.notifications.irc_backend import IrcBackend
from awx.main.fields import JSONField
logger = logging.getLogger('awx.main.models.notifications')
@@ -70,12 +69,12 @@ class NotificationTemplate(CommonModelNameNotUnique):
choices=NOTIFICATION_TYPE_CHOICES,
)
notification_configuration = prevent_search(JSONField(blank=False))
notification_configuration = prevent_search(models.JSONField(default=dict))
def default_messages():
return {'started': None, 'success': None, 'error': None, 'workflow_approval': None}
messages = JSONField(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.'))
messages = models.JSONField(null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.'))
def has_message(self, condition):
potential_template = self.messages.get(condition, {})
@@ -237,7 +236,7 @@ class Notification(CreatedModifiedModel):
default='',
editable=False,
)
body = JSONField(blank=True)
body = models.JSONField(default=dict, null=True, blank=True)
def get_absolute_url(self, request=None):
return reverse('api:notification_detail', kwargs={'pk': self.pk}, request=request)

9
awx/main/models/projects.py
View File

@@ -38,7 +38,6 @@ from awx.main.models.rbac import (
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
ROLE_SINGLETON_SYSTEM_AUDITOR,
)
from awx.main.fields import JSONField
__all__ = ['Project', 'ProjectUpdate']
@@ -294,17 +293,17 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
help_text=_('The last revision fetched by a project update'),
)
playbook_files = JSONField(
playbook_files = models.JSONField(
default=list,
blank=True,
default=[],
editable=False,
verbose_name=_('Playbook Files'),
help_text=_('List of playbooks found in the project'),
)
inventory_files = JSONField(
inventory_files = models.JSONField(
default=list,
blank=True,
default=[],
editable=False,
verbose_name=_('Inventory Files'),
help_text=_('Suggested list of content that could be Ansible inventory in the project'),

9
awx/main/models/unified_jobs.py
View File

@@ -54,7 +54,7 @@ from awx.main.utils import polymorphic
from awx.main.constants import ACTIVE_STATES, CAN_CANCEL
from awx.main.redact import UriCleaner, REPLACE_STR
from awx.main.consumers import emit_channel_notification
from awx.main.fields import JSONField, JSONBField, AskForField, OrderedManyToManyField
from awx.main.fields import AskForField, OrderedManyToManyField
__all__ = ['UnifiedJobTemplate', 'UnifiedJob', 'StdoutMaxBytesExceeded']
@@ -653,9 +653,10 @@ class UnifiedJob(
editable=False,
)
job_env = prevent_search(
JSONField(
blank=True,
models.JSONField(
default=dict,
null=True,
blank=True,
editable=False,
)
)
@@ -704,7 +705,7 @@ class UnifiedJob(
'Credential',
related_name='%(class)ss',
)
installed_collections = JSONBField(
installed_collections = models.JSONField(
blank=True,
default=dict,
editable=False,

6
awx/main/models/workflow.py
View File

@@ -40,7 +40,6 @@ from awx.main.models.mixins import (
from awx.main.models.jobs import LaunchTimeConfigBase, LaunchTimeConfig, JobTemplate
from awx.main.models.credential import Credential
from awx.main.redact import REPLACE_STR
from awx.main.fields import JSONField
from awx.main.utils import schedule_task_manager
@@ -232,9 +231,10 @@ class WorkflowJobNode(WorkflowNodeBase):
default=None,
on_delete=models.CASCADE,
)
ancestor_artifacts = JSONField(
blank=True,
ancestor_artifacts = models.JSONField(
default=dict,
null=True,
blank=True,
editable=False,
)
do_not_run = models.BooleanField(

8
awx/main/tests/factories/fixtures.py
View File

@@ -180,8 +180,8 @@ def mk_job_template(
jt.project = project
jt.survey_spec = spec
if jt.survey_spec is not None:
if spec is not None:
jt.survey_spec = spec
jt.survey_enabled = True
if persisted:
@@ -212,8 +212,8 @@ def mk_workflow_job_template(name, extra_vars='', spec=None, organization=None,
wfjt = WorkflowJobTemplate(name=name, extra_vars=extra_vars, organization=organization, webhook_service=webhook_service)
wfjt.survey_spec = spec
if wfjt.survey_spec:
if spec:
wfjt.survey_spec = spec
wfjt.survey_enabled = True
if persisted:

6
awx/main/tests/functional/conftest.py
View File

@@ -15,7 +15,6 @@ from django.core.serializers.json import DjangoJSONEncoder
from django.db.backends.sqlite3.base import SQLiteCursorWrapper
# AWX
from awx.main.fields import JSONBField
from awx.main.models.projects import Project
from awx.main.models.ha import Instance
@@ -755,11 +754,6 @@ def get_db_prep_save(self, value, connection, **kwargs):
return value
@pytest.fixture
def monkeypatch_jsonbfield_get_db_prep_save(mocker):
JSONBField.get_db_prep_save = get_db_prep_save
@pytest.fixture
def oauth_application(admin):
return Application.objects.create(name='test app', user=admin, client_type='confidential', authorization_grant_type='password')

34
awx/main/tests/unit/utils/test_filters.py
View File

@@ -4,7 +4,6 @@ from unittest import mock
# AWX
from awx.main.utils.filters import SmartFilter, ExternalLoggerEnabled
from awx.main.models import Host
# Django
from django.db.models import Q
@@ -219,39 +218,6 @@ class TestSmartFilterQueryFromString:
assert str(q) == str(q_expected)
class TestSmartFilterQueryFromStringNoDB:
@pytest.mark.parametrize(
"filter_string,q_expected",
[
(
'ansible_facts__a="true" and ansible_facts__b="true" and ansible_facts__c="true"',
(
Q(**{u"ansible_facts__contains": {u"a": u"true"}})
& Q(**{u"ansible_facts__contains": {u"b": u"true"}})
& Q(**{u"ansible_facts__contains": {u"c": u"true"}})
),
),
(
'ansible_facts__a="true" or ansible_facts__b="true" or ansible_facts__c="true"',
(
Q(**{u"ansible_facts__contains": {u"a": u"true"}})
| Q(**{u"ansible_facts__contains": {u"b": u"true"}})
| Q(**{u"ansible_facts__contains": {u"c": u"true"}})
),
),
('search=foo', Q(Q(**{u"description__icontains": u"foo"}) | Q(**{u"name__icontains": u"foo"}))),
(
'search=foo and ansible_facts__a="null"',
Q(Q(**{u"description__icontains": u"foo"}) | Q(**{u"name__icontains": u"foo"})) & Q(**{u"ansible_facts__contains": {u"a": u"\"null\""}}),
),
('name=foo or name=bar and name=foobar', Q(name="foo") | Q(name="bar") & Q(name="foobar")),
],
)
def test_does_not_invoke_db(self, filter_string, q_expected):
q = SmartFilter.query_from_string(filter_string)
assert str(q.query) == str(Host.objects.filter(q_expected).query)
'''
#('"facts__quoted_val"="f\"oo"', 1),
#('facts__facts__arr[]="foo"', 1),

4
awx/main/utils/filters.py
View File

@@ -188,13 +188,11 @@ class SmartFilter(object):
'''
def _json_path_to_contains(self, k, v):
from awx.main.fields import JSONBField # avoid a circular import
if not k.startswith(SmartFilter.SEARCHABLE_RELATIONSHIP):
v = self.strip_quotes_traditional_logic(v)
return (k, v)
for match in JSONBField.get_lookups().keys():
for match in models.JSONField.get_lookups().keys():
match = '__{}'.format(match)
if k.endswith(match):
if match == '__exact':

11
requirements/README.md
View File

@@ -105,17 +105,6 @@ Upgrading to 4.0.0 causes error because imports changed.
ImportError: cannot import name 'KeyVaultClient'
```
### django-jsonfield
Instead of calling a `loads()` operation, the returned value is casted into
a string in some cases, introduced in the change:
https://github.com/adamchainz/django-jsonfield/pull/14
This breaks a very large amount of AWX code that assumes these fields
are returned as dicts. Upgrading this library will require a refactor
to accommodate this change.
### pip and setuptools
The offline installer needs to have functionality confirmed before upgrading these.