From 239c60a2bdd3193bc211779def7a8484025a3fa0 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 31 May 2018 15:43:58 -0400 Subject: [PATCH] fix tracebacks when running migrations --- .../0010_v322_add_ovirt4_tower_inventory.py | 2 + .../migrations/0012_v322_update_cred_types.py | 2 + .../migrations/0013_v330_multi_credential.py | 9 +-- .../0023_v330_inventory_multicred.py | 4 +- awx/main/migrations/_credentialtypes.py | 11 +++ awx/main/migrations/_multi_cred.py | 72 ++++++++++++++++++- awx/main/signals.py | 26 ++++--- 7 files changed, 109 insertions(+), 17 deletions(-) diff --git a/awx/main/migrations/0010_v322_add_ovirt4_tower_inventory.py b/awx/main/migrations/0010_v322_add_ovirt4_tower_inventory.py index aac423cd1c..2cdf557856 100644 --- a/awx/main/migrations/0010_v322_add_ovirt4_tower_inventory.py +++ b/awx/main/migrations/0010_v322_add_ovirt4_tower_inventory.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals # AWX +from awx.main.migrations import _migration_utils as migration_utils from awx.main.migrations import _credentialtypes as credentialtypes from django.db import migrations, models @@ -14,6 +15,7 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RunPython(migration_utils.set_current_apps_for_migrations), migrations.RunPython(credentialtypes.create_rhv_tower_credtype), migrations.AlterField( model_name='inventorysource', diff --git a/awx/main/migrations/0012_v322_update_cred_types.py b/awx/main/migrations/0012_v322_update_cred_types.py index 86d9fd55fa..b1e77fd810 100644 --- a/awx/main/migrations/0012_v322_update_cred_types.py +++ b/awx/main/migrations/0012_v322_update_cred_types.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals # AWX +from awx.main.migrations import _migration_utils as migration_utils from awx.main.migrations import _credentialtypes as credentialtypes from django.db import migrations @@ -14,5 +15,6 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RunPython(migration_utils.set_current_apps_for_migrations), migrations.RunPython(credentialtypes.add_azure_cloud_environment_field), ] diff --git a/awx/main/migrations/0013_v330_multi_credential.py b/awx/main/migrations/0013_v330_multi_credential.py index abc186c382..a5f00fdf08 100644 --- a/awx/main/migrations/0013_v330_multi_credential.py +++ b/awx/main/migrations/0013_v330_multi_credential.py @@ -5,7 +5,7 @@ from django.db import migrations, models from awx.main.migrations import _migration_utils as migration_utils from awx.main.migrations import _credentialtypes as credentialtypes -from awx.main.migrations._multi_cred import migrate_to_multi_cred +from awx.main.migrations._multi_cred import migrate_to_multi_cred, migrate_back_from_multi_cred class Migration(migrations.Migration): @@ -25,8 +25,8 @@ class Migration(migrations.Migration): name='credentials', field=models.ManyToManyField(related_name='unifiedjobtemplates', to='main.Credential'), ), - migrations.RunPython(migration_utils.set_current_apps_for_migrations), - migrations.RunPython(migrate_to_multi_cred), + migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrate_back_from_multi_cred), + migrations.RunPython(migrate_to_multi_cred, migration_utils.set_current_apps_for_migrations), migrations.RemoveField( model_name='job', name='credential', @@ -51,5 +51,6 @@ class Migration(migrations.Migration): model_name='jobtemplate', name='vault_credential', ), - migrations.RunPython(credentialtypes.add_vault_id_field) + migrations.RunPython(migration_utils.set_current_apps_for_migrations, credentialtypes.remove_vault_id_field), + migrations.RunPython(credentialtypes.add_vault_id_field, migration_utils.set_current_apps_for_migrations) ] diff --git a/awx/main/migrations/0023_v330_inventory_multicred.py b/awx/main/migrations/0023_v330_inventory_multicred.py index b184abc296..fdb95e8ddc 100644 --- a/awx/main/migrations/0023_v330_inventory_multicred.py +++ b/awx/main/migrations/0023_v330_inventory_multicred.py @@ -19,8 +19,8 @@ class Migration(migrations.Migration): operations = [ # Run data migration before removing the old credential field - migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrations.RunPython.noop), - migrations.RunPython(migrate_inventory_source_cred, migrate_inventory_source_cred_reverse), + migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrate_inventory_source_cred_reverse), + migrations.RunPython(migrate_inventory_source_cred, migration_utils.set_current_apps_for_migrations), migrations.RemoveField( model_name='inventorysource', name='credential', diff --git a/awx/main/migrations/_credentialtypes.py b/awx/main/migrations/_credentialtypes.py index fbf812e8c2..9d78cec49d 100644 --- a/awx/main/migrations/_credentialtypes.py +++ b/awx/main/migrations/_credentialtypes.py @@ -180,6 +180,17 @@ def add_vault_id_field(apps, schema_editor): vault_credtype.save() +def remove_vault_id_field(apps, schema_editor): + vault_credtype = CredentialType.objects.get(kind='vault') + idx = 0 + for i, input in enumerate(vault_credtype.inputs['fields']): + if input['id'] == 'vault_id': + idx = i + break + vault_credtype.inputs['fields'].pop(idx) + vault_credtype.save() + + def create_rhv_tower_credtype(apps, schema_editor): CredentialType.setup_tower_managed_defaults() diff --git a/awx/main/migrations/_multi_cred.py b/awx/main/migrations/_multi_cred.py index 5e4ed5ff4b..1b2a99c110 100644 --- a/awx/main/migrations/_multi_cred.py +++ b/awx/main/migrations/_multi_cred.py @@ -1,56 +1,124 @@ +import logging + + +logger = logging.getLogger('awx.main.migrations') + + def migrate_to_multi_cred(app, schema_editor): Job = app.get_model('main', 'Job') JobTemplate = app.get_model('main', 'JobTemplate') + ct = 0 for cls in (Job, JobTemplate): for j in cls.objects.iterator(): if j.credential: + ct += 1 + logger.debug('Migrating cred %s to %s %s multi-cred relation.', j.credential_id, cls, j.id) j.credentials.add(j.credential) if j.vault_credential: + ct += 1 + logger.debug('Migrating cred %s to %s %s multi-cred relation.', j.vault_credential_id, cls, j.id) j.credentials.add(j.vault_credential) for cred in j.extra_credentials.all(): + ct += 1 + logger.debug('Migrating cred %s to %s %s multi-cred relation.', cred.id, cls, j.id) j.credentials.add(cred) + if ct: + logger.info('Finished migrating %s credentials to multi-cred', ct) + + +def migrate_back_from_multi_cred(app, schema_editor): + Job = app.get_model('main', 'Job') + JobTemplate = app.get_model('main', 'JobTemplate') + CredentialType = app.get_model('main', 'CredentialType') + vault_credtype = CredentialType.objects.get(kind='vault') + ssh_credtype = CredentialType.objects.get(kind='ssh') + + ct = 0 + for cls in (Job, JobTemplate): + for j in cls.objects.iterator(): + for cred in j.credentials.iterator(): + changed = False + if cred.credential_type_id == vault_credtype.id: + changed = True + ct += 1 + logger.debug('Reverse migrating vault cred %s for %s %s', cred.id, cls, j.id) + j.vault_credential = cred + elif cred.credential_type_id == ssh_credtype.id: + changed = True + ct += 1 + logger.debug('Reverse migrating ssh cred %s for %s %s', cred.id, cls, j.id) + j.credential = cred + else: + changed = True + ct += 1 + logger.debug('Reverse migrating cloud cred %s for %s %s', cred.id, cls, j.id) + j.extra_credentials.add(cred) + if changed: + j.save() + if ct: + logger.info('Finished reverse migrating %s credentials from multi-cred', ct) def migrate_workflow_cred(app, schema_editor): WorkflowJobTemplateNode = app.get_model('main', 'WorkflowJobTemplateNode') WorkflowJobNode = app.get_model('main', 'WorkflowJobNode') + ct = 0 for cls in (WorkflowJobNode, WorkflowJobTemplateNode): for node in cls.objects.iterator(): if node.credential: - node.credentials.add(j.credential) + logger.debug('Migrating prompted credential %s for %s %s', node.credential_id, cls, node.id) + ct += 1 + node.credentials.add(node.credential) + if ct: + logger.info('Finished migrating total of %s workflow prompted credentials', ct) def migrate_workflow_cred_reverse(app, schema_editor): WorkflowJobTemplateNode = app.get_model('main', 'WorkflowJobTemplateNode') WorkflowJobNode = app.get_model('main', 'WorkflowJobNode') + ct = 0 for cls in (WorkflowJobNode, WorkflowJobTemplateNode): for node in cls.objects.iterator(): cred = node.credentials.first() if cred: node.credential = cred - node.save() + logger.debug('Reverse migrating prompted credential %s for %s %s', node.credential_id, cls, node.id) + ct += 1 + node.save(update_fields=['credential']) + if ct: + logger.info('Finished reverse migrating total of %s workflow prompted credentials', ct) def migrate_inventory_source_cred(app, schema_editor): InventoryUpdate = app.get_model('main', 'InventoryUpdate') InventorySource = app.get_model('main', 'InventorySource') + ct = 0 for cls in (InventoryUpdate, InventorySource): for obj in cls.objects.iterator(): if obj.credential: + ct += 1 + logger.debug('Migrating credential %s for %s %s', obj.credential_id, cls, obj.id) obj.credentials.add(obj.credential) + if ct: + logger.info('Finished migrating %s inventory source credentials to multi-cred', ct) def migrate_inventory_source_cred_reverse(app, schema_editor): InventoryUpdate = app.get_model('main', 'InventoryUpdate') InventorySource = app.get_model('main', 'InventorySource') + ct = 0 for cls in (InventoryUpdate, InventorySource): for obj in cls.objects.iterator(): cred = obj.credentials.first() if cred: + ct += 1 + logger.debug('Reverse migrating credential %s for %s %s', cred.id, cls, obj.id) obj.credential = cred obj.save() + if ct: + logger.info('Finished reverse migrating %s inventory source credentials from multi-cred', ct) diff --git a/awx/main/signals.py b/awx/main/signals.py index 11d192e6e9..4bf8e38c8b 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -6,6 +6,7 @@ import contextlib import logging import threading import json +import sys # Django from django.conf import settings @@ -31,7 +32,7 @@ from awx.main.models import * # noqa from django.contrib.sessions.models import Session from awx.api.serializers import * # noqa from awx.main.constants import TOKEN_CENSOR -from awx.main.utils import model_instance_diff, model_to_dict, camelcase_to_underscore +from awx.main.utils import model_instance_diff, model_to_dict, camelcase_to_underscore, get_current_apps from awx.main.utils import ignore_inventory_computed_fields, ignore_inventory_group_removal, _inventory_updates from awx.main.tasks import update_inventory_computed_fields from awx.main.fields import ( @@ -51,6 +52,13 @@ logger = logging.getLogger('awx.main.signals') # when a Host-Group or Group-Group relationship is updated, or when a Job is deleted +def get_activity_stream_class(): + if 'migrate' in sys.argv: + return get_current_apps().get_model('main', 'ActivityStream') + else: + return ActivityStream + + def get_current_user_or_none(): u = get_current_user() if not isinstance(u, User): @@ -418,7 +426,7 @@ def activity_stream_create(sender, instance, created, **kwargs): changes['extra_vars'] = instance.display_extra_vars() if type(instance) == OAuth2AccessToken: changes['token'] = TOKEN_CENSOR - activity_entry = ActivityStream( + activity_entry = get_activity_stream_class()( operation='create', object1=object1, changes=json.dumps(changes), @@ -428,7 +436,7 @@ def activity_stream_create(sender, instance, created, **kwargs): # we don't really use them anyway. if instance._meta.model_name != 'setting': # Is not conf.Setting instance activity_entry.save() - getattr(activity_entry, object1).add(instance) + getattr(activity_entry, object1).add(instance.pk) else: activity_entry.setting = conf_to_dict(instance) activity_entry.save() @@ -452,14 +460,14 @@ def activity_stream_update(sender, instance, **kwargs): if getattr(_type, '_deferred', False): return object1 = camelcase_to_underscore(instance.__class__.__name__) - activity_entry = ActivityStream( + activity_entry = get_activity_stream_class()( operation='update', object1=object1, changes=json.dumps(changes), actor=get_current_user_or_none()) if instance._meta.model_name != 'setting': # Is not conf.Setting instance activity_entry.save() - getattr(activity_entry, object1).add(instance) + getattr(activity_entry, object1).add(instance.pk) else: activity_entry.setting = conf_to_dict(instance) activity_entry.save() @@ -485,7 +493,7 @@ def activity_stream_delete(sender, instance, **kwargs): object1 = camelcase_to_underscore(instance.__class__.__name__) if type(instance) == OAuth2AccessToken: changes['token'] = TOKEN_CENSOR - activity_entry = ActivityStream( + activity_entry = get_activity_stream_class()( operation='delete', changes=json.dumps(changes), object1=object1, @@ -532,7 +540,7 @@ def activity_stream_associate(sender, instance, **kwargs): continue if isinstance(obj1, SystemJob) or isinstance(obj2_actual, SystemJob): continue - activity_entry = ActivityStream( + activity_entry = get_activity_stream_class()( changes=json.dumps(dict(object1=object1, object1_pk=obj1.pk, object2=object2, @@ -545,8 +553,8 @@ def activity_stream_associate(sender, instance, **kwargs): object_relationship_type=obj_rel, actor=get_current_user_or_none()) activity_entry.save() - getattr(activity_entry, object1).add(obj1) - getattr(activity_entry, object2).add(obj2_actual) + getattr(activity_entry, object1).add(obj1.pk) + getattr(activity_entry, object2).add(obj2_actual.pk) # Record the role for RBAC changes if 'role' in kwargs: