Merge pull request #2013 from AlanCoding/big_trouble_in_0013

Fix tracebacks when running migrations

awx/main/migrations/0010_v322_add_ovirt4_tower_inventory.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 # AWX
+from awx.main.migrations import _migration_utils as migration_utils
 from awx.main.migrations import _credentialtypes as credentialtypes
 
 from django.db import migrations, models
@@ -14,6 +15,7 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.RunPython(migration_utils.set_current_apps_for_migrations),
         migrations.RunPython(credentialtypes.create_rhv_tower_credtype),
         migrations.AlterField(
             model_name='inventorysource',

awx/main/migrations/0012_v322_update_cred_types.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 # AWX
+from awx.main.migrations import _migration_utils as migration_utils
 from awx.main.migrations import _credentialtypes as credentialtypes
 
 from django.db import migrations
@@ -14,5 +15,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.RunPython(migration_utils.set_current_apps_for_migrations),
         migrations.RunPython(credentialtypes.add_azure_cloud_environment_field),
     ]

awx/main/migrations/0013_v330_multi_credential.py

@@ -5,7 +5,7 @@ from django.db import migrations, models
 
 from awx.main.migrations import _migration_utils as migration_utils
 from awx.main.migrations import _credentialtypes as credentialtypes
-from awx.main.migrations._multi_cred import migrate_to_multi_cred
+from awx.main.migrations._multi_cred import migrate_to_multi_cred, migrate_back_from_multi_cred
 
 
 class Migration(migrations.Migration):
@@ -25,8 +25,8 @@ class Migration(migrations.Migration):
             name='credentials',
             field=models.ManyToManyField(related_name='unifiedjobtemplates', to='main.Credential'),
         ),
-        migrations.RunPython(migration_utils.set_current_apps_for_migrations),
+        migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrate_back_from_multi_cred),
-        migrations.RunPython(migrate_to_multi_cred),
+        migrations.RunPython(migrate_to_multi_cred, migration_utils.set_current_apps_for_migrations),
         migrations.RemoveField(
             model_name='job',
             name='credential',
@@ -51,5 +51,6 @@ class Migration(migrations.Migration):
             model_name='jobtemplate',
             name='vault_credential',
         ),
-        migrations.RunPython(credentialtypes.add_vault_id_field)
+        migrations.RunPython(migration_utils.set_current_apps_for_migrations, credentialtypes.remove_vault_id_field),
+        migrations.RunPython(credentialtypes.add_vault_id_field, migration_utils.set_current_apps_for_migrations)
     ]

awx/main/migrations/0023_v330_inventory_multicred.py

@@ -19,8 +19,8 @@ class Migration(migrations.Migration):
 
     operations = [
         # Run data migration before removing the old credential field
-        migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrations.RunPython.noop),
+        migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrate_inventory_source_cred_reverse),
-        migrations.RunPython(migrate_inventory_source_cred, migrate_inventory_source_cred_reverse),
+        migrations.RunPython(migrate_inventory_source_cred, migration_utils.set_current_apps_for_migrations),
         migrations.RemoveField(
             model_name='inventorysource',
             name='credential',

awx/main/migrations/_credentialtypes.py

@@ -180,6 +180,17 @@ def add_vault_id_field(apps, schema_editor):
     vault_credtype.save()
 
 
+def remove_vault_id_field(apps, schema_editor):
+    vault_credtype = CredentialType.objects.get(kind='vault')
+    idx = 0
+    for i, input in enumerate(vault_credtype.inputs['fields']):
+        if input['id'] == 'vault_id':
+            idx = i
+            break
+    vault_credtype.inputs['fields'].pop(idx)
+    vault_credtype.save()
+
+
 def create_rhv_tower_credtype(apps, schema_editor):
     CredentialType.setup_tower_managed_defaults()
 

awx/main/migrations/_multi_cred.py

@@ -1,56 +1,124 @@
+import logging
+
+
+logger = logging.getLogger('awx.main.migrations')
+
+
 def migrate_to_multi_cred(app, schema_editor):
     Job = app.get_model('main', 'Job')
     JobTemplate = app.get_model('main', 'JobTemplate')
 
+    ct = 0
     for cls in (Job, JobTemplate):
         for j in cls.objects.iterator():
             if j.credential:
+                ct += 1
+                logger.debug('Migrating cred %s to %s %s multi-cred relation.', j.credential_id, cls, j.id)
                 j.credentials.add(j.credential)
             if j.vault_credential:
+                ct += 1
+                logger.debug('Migrating cred %s to %s %s multi-cred relation.', j.vault_credential_id, cls, j.id)
                 j.credentials.add(j.vault_credential)
             for cred in j.extra_credentials.all():
+                ct += 1
+                logger.debug('Migrating cred %s to %s %s multi-cred relation.', cred.id, cls, j.id)
                 j.credentials.add(cred)
+    if ct:
+        logger.info('Finished migrating %s credentials to multi-cred', ct)
+
+
+def migrate_back_from_multi_cred(app, schema_editor):
+    Job = app.get_model('main', 'Job')
+    JobTemplate = app.get_model('main', 'JobTemplate')
+    CredentialType = app.get_model('main', 'CredentialType')
+    vault_credtype = CredentialType.objects.get(kind='vault')
+    ssh_credtype = CredentialType.objects.get(kind='ssh')
+
+    ct = 0
+    for cls in (Job, JobTemplate):
+        for j in cls.objects.iterator():
+            for cred in j.credentials.iterator():
+                changed = False
+                if cred.credential_type_id == vault_credtype.id:
+                    changed = True
+                    ct += 1
+                    logger.debug('Reverse migrating vault cred %s for %s %s', cred.id, cls, j.id)
+                    j.vault_credential = cred
+                elif cred.credential_type_id == ssh_credtype.id:
+                    changed = True
+                    ct += 1
+                    logger.debug('Reverse migrating ssh cred %s for %s %s', cred.id, cls, j.id)
+                    j.credential = cred
+                else:
+                    changed = True
+                    ct += 1
+                    logger.debug('Reverse migrating cloud cred %s for %s %s', cred.id, cls, j.id)
+                    j.extra_credentials.add(cred)
+                if changed:
+                    j.save()
+    if ct:
+        logger.info('Finished reverse migrating %s credentials from multi-cred', ct)
 
 
 def migrate_workflow_cred(app, schema_editor):
     WorkflowJobTemplateNode = app.get_model('main', 'WorkflowJobTemplateNode')
     WorkflowJobNode = app.get_model('main', 'WorkflowJobNode')
 
+    ct = 0
     for cls in (WorkflowJobNode, WorkflowJobTemplateNode):
         for node in cls.objects.iterator():
             if node.credential:
-                node.credentials.add(j.credential)
+                logger.debug('Migrating prompted credential %s for %s %s', node.credential_id, cls, node.id)
+                ct += 1
+                node.credentials.add(node.credential)
+    if ct:
+        logger.info('Finished migrating total of %s workflow prompted credentials', ct)
 
 
 def migrate_workflow_cred_reverse(app, schema_editor):
     WorkflowJobTemplateNode = app.get_model('main', 'WorkflowJobTemplateNode')
     WorkflowJobNode = app.get_model('main', 'WorkflowJobNode')
 
+    ct = 0
     for cls in (WorkflowJobNode, WorkflowJobTemplateNode):
         for node in cls.objects.iterator():
             cred = node.credentials.first()
             if cred:
                 node.credential = cred
-                node.save()
+                logger.debug('Reverse migrating prompted credential %s for %s %s', node.credential_id, cls, node.id)
+                ct += 1
+                node.save(update_fields=['credential'])
+    if ct:
+        logger.info('Finished reverse migrating total of %s workflow prompted credentials', ct)
 
 
 def migrate_inventory_source_cred(app, schema_editor):
     InventoryUpdate = app.get_model('main', 'InventoryUpdate')
     InventorySource = app.get_model('main', 'InventorySource')
 
+    ct = 0
     for cls in (InventoryUpdate, InventorySource):
         for obj in cls.objects.iterator():
             if obj.credential:
+                ct += 1
+                logger.debug('Migrating credential %s for %s %s', obj.credential_id, cls, obj.id)
                 obj.credentials.add(obj.credential)
+    if ct:
+        logger.info('Finished migrating %s inventory source credentials to multi-cred', ct)
 
 
 def migrate_inventory_source_cred_reverse(app, schema_editor):
     InventoryUpdate = app.get_model('main', 'InventoryUpdate')
     InventorySource = app.get_model('main', 'InventorySource')
 
+    ct = 0
     for cls in (InventoryUpdate, InventorySource):
         for obj in cls.objects.iterator():
             cred = obj.credentials.first()
             if cred:
+                ct += 1
+                logger.debug('Reverse migrating credential %s for %s %s', cred.id, cls, obj.id)
                 obj.credential = cred
                 obj.save()
+    if ct:
+        logger.info('Finished reverse migrating %s inventory source credentials from multi-cred', ct)

awx/main/signals.py

@@ -6,6 +6,7 @@ import contextlib
 import logging
 import threading
 import json
+import sys
 
 # Django
 from django.conf import settings
@@ -31,7 +32,7 @@ from awx.main.models import * # noqa
 from django.contrib.sessions.models import Session
 from awx.api.serializers import * # noqa
 from awx.main.constants import TOKEN_CENSOR
-from awx.main.utils import model_instance_diff, model_to_dict, camelcase_to_underscore
+from awx.main.utils import model_instance_diff, model_to_dict, camelcase_to_underscore, get_current_apps
 from awx.main.utils import ignore_inventory_computed_fields, ignore_inventory_group_removal, _inventory_updates
 from awx.main.tasks import update_inventory_computed_fields
 from awx.main.fields import (
@@ -51,6 +52,13 @@ logger = logging.getLogger('awx.main.signals')
 # when a Host-Group or Group-Group relationship is updated, or when a Job is deleted
 
 
+def get_activity_stream_class():
+    if 'migrate' in sys.argv:
+        return get_current_apps().get_model('main', 'ActivityStream')
+    else:
+        return ActivityStream
+
+
 def get_current_user_or_none():
     u = get_current_user()
     if not isinstance(u, User):
@@ -418,7 +426,7 @@ def activity_stream_create(sender, instance, created, **kwargs):
                 changes['extra_vars'] = instance.display_extra_vars()
         if type(instance) == OAuth2AccessToken:
             changes['token'] = TOKEN_CENSOR
-        activity_entry = ActivityStream(
+        activity_entry = get_activity_stream_class()(
             operation='create',
             object1=object1,
             changes=json.dumps(changes),
@@ -428,7 +436,7 @@ def activity_stream_create(sender, instance, created, **kwargs):
         #      we don't really use them anyway.
         if instance._meta.model_name != 'setting':  # Is not conf.Setting instance
             activity_entry.save()
-            getattr(activity_entry, object1).add(instance)
+            getattr(activity_entry, object1).add(instance.pk)
         else:
             activity_entry.setting = conf_to_dict(instance)
             activity_entry.save()
@@ -452,14 +460,14 @@ def activity_stream_update(sender, instance, **kwargs):
     if getattr(_type, '_deferred', False):
         return
     object1 = camelcase_to_underscore(instance.__class__.__name__)
-    activity_entry = ActivityStream(
+    activity_entry = get_activity_stream_class()(
         operation='update',
         object1=object1,
         changes=json.dumps(changes),
         actor=get_current_user_or_none())
     if instance._meta.model_name != 'setting':  # Is not conf.Setting instance
         activity_entry.save()
-        getattr(activity_entry, object1).add(instance)
+        getattr(activity_entry, object1).add(instance.pk)
     else:
         activity_entry.setting = conf_to_dict(instance)
         activity_entry.save()
@@ -485,7 +493,7 @@ def activity_stream_delete(sender, instance, **kwargs):
     object1 = camelcase_to_underscore(instance.__class__.__name__)
     if type(instance) == OAuth2AccessToken:
         changes['token'] = TOKEN_CENSOR
-    activity_entry = ActivityStream(
+    activity_entry = get_activity_stream_class()(
         operation='delete',
         changes=json.dumps(changes),
         object1=object1,
@@ -532,7 +540,7 @@ def activity_stream_associate(sender, instance, **kwargs):
                 continue
             if isinstance(obj1, SystemJob) or isinstance(obj2_actual, SystemJob):
                 continue
-            activity_entry = ActivityStream(
+            activity_entry = get_activity_stream_class()(
                 changes=json.dumps(dict(object1=object1,
                                         object1_pk=obj1.pk,
                                         object2=object2,
@@ -545,8 +553,8 @@ def activity_stream_associate(sender, instance, **kwargs):
                 object_relationship_type=obj_rel,
                 actor=get_current_user_or_none())
             activity_entry.save()
-            getattr(activity_entry, object1).add(obj1)
+            getattr(activity_entry, object1).add(obj1.pk)
-            getattr(activity_entry, object2).add(obj2_actual)
+            getattr(activity_entry, object2).add(obj2_actual.pk)
 
             # Record the role for RBAC changes
             if 'role' in kwargs: