diff --git a/awx/main/management/commands/cleanup_deleted.py b/awx/main/management/commands/cleanup_deleted.py index b6fd5360e5..7fd726be7f 100644 --- a/awx/main/management/commands/cleanup_deleted.py +++ b/awx/main/management/commands/cleanup_deleted.py @@ -111,8 +111,6 @@ class Command(BaseCommand): n_deleted_items = 0 n_deleted_items += self.cleanup_model(User) - for model in self.get_models(PrimordialModel): - n_deleted_items += self.cleanup_model(model) if not self.dry_run: self.logger.log(99, "Removed %d items", n_deleted_items) diff --git a/awx/main/migrations/0005_v300_active_flag_removal.py b/awx/main/migrations/0005_v300_active_flag_removal.py new file mode 100644 index 0000000000..adc6a0cddf --- /dev/null +++ b/awx/main/migrations/0005_v300_active_flag_removal.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from awx.main.migrations import _cleanup_deleted as cleanup_deleted +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0004_v300_changes'), + ] + + operations = [ + migrations.RunPython(cleanup_deleted.cleanup_deleted), + + migrations.RemoveField( + model_name='credential', + name='active', + ), + migrations.RemoveField( + model_name='custominventoryscript', + name='active', + ), + migrations.RemoveField( + model_name='group', + name='active', + ), + migrations.RemoveField( + model_name='host', + name='active', + ), + migrations.RemoveField( + model_name='inventory', + name='active', + ), + migrations.RemoveField( + model_name='notifier', + name='active', + ), + migrations.RemoveField( + model_name='organization', + name='active', + ), + migrations.RemoveField( + model_name='permission', + name='active', + ), + migrations.RemoveField( + model_name='schedule', + name='active', + ), + migrations.RemoveField( + model_name='team', + name='active', + ), + migrations.RemoveField( + model_name='unifiedjob', + name='active', + ), + migrations.RemoveField( + model_name='unifiedjobtemplate', + name='active', + ), + ] diff --git a/awx/main/migrations/0006_v300_active_flag_removal.py b/awx/main/migrations/0006_v300_active_flag_removal.py deleted file mode 100644 index 795db642b2..0000000000 --- a/awx/main/migrations/0006_v300_active_flag_removal.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from awx.main.migrations import _rbac as rbac -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0005_v300_changes'), - ] - - operations = [ - # This is a placeholder for our future active flag removal work - ] diff --git a/awx/main/migrations/0005_v300_changes.py b/awx/main/migrations/0006_v300_changes.py similarity index 98% rename from awx/main/migrations/0005_v300_changes.py rename to awx/main/migrations/0006_v300_changes.py index e350c881f2..4162ff3bd1 100644 --- a/awx/main/migrations/0005_v300_changes.py +++ b/awx/main/migrations/0006_v300_changes.py @@ -107,7 +107,7 @@ def create_system_job_templates(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('main', '0004_v300_changes'), + ('main', '0005_v300_active_flag_removal'), ] operations = [ diff --git a/awx/main/migrations/0007_v300_rbac_changes.py b/awx/main/migrations/0007_v300_rbac_changes.py index f77d4c026a..b1aae399d3 100644 --- a/awx/main/migrations/0007_v300_rbac_changes.py +++ b/awx/main/migrations/0007_v300_rbac_changes.py @@ -14,7 +14,7 @@ class Migration(migrations.Migration): ('taggit', '0002_auto_20150616_2121'), ('contenttypes', '0002_remove_content_type_name'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('main', '0006_v300_active_flag_removal'), + ('main', '0006_v300_changes'), ] operations = [ diff --git a/awx/main/migrations/_cleanup_deleted.py b/awx/main/migrations/_cleanup_deleted.py new file mode 100644 index 0000000000..db187efeb4 --- /dev/null +++ b/awx/main/migrations/_cleanup_deleted.py @@ -0,0 +1,85 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +# Python +import logging + +# Django +from django.db import transaction +from django.utils.dateparse import parse_datetime + +def cleanup_deleted(apps, schema_editor): + logger = logging.getLogger('awx.main.migrations.cleanup_deleted') + + def cleanup_model(model): + + ''' + Presume the '_deleted_' string to be in the 'name' field unless considering the User model. + When considering the User model, presume the '_d_' string to be in the 'username' field. + ''' + logger.debug('cleaning up model %s', model) + + name_field = 'name' + name_prefix = '_deleted_' + active_field = None + n_deleted_items = 0 + for field in model._meta.fields: + if field.name in ('is_active', 'active'): + active_field = field.name + if field.name == 'is_active': # is User model + name_field = 'username' + name_prefix = '_d_' + if not active_field: + logger.warning('skipping model %s, no active field', model) + return n_deleted_items + qs = model.objects.filter(**{ + active_field: False, + '%s__startswith' % name_field: name_prefix, + }) + pks_to_delete = set() + for instance in qs.iterator(): + dt = parse_datetime(getattr(instance, name_field).split('_')[2]) + if not dt: + logger.warning('unable to find deleted timestamp in %s field', name_field) + else: + action_text = 'deleting' + logger.info('%s %s', action_text, instance) + n_deleted_items += 1 + instance.delete() + + # Cleanup objects in batches instead of deleting each one individually. + if len(pks_to_delete) >= 50: + model.objects.filter(pk__in=pks_to_delete).delete() + pks_to_delete.clear() + if len(pks_to_delete): + model.objects.filter(pk__in=pks_to_delete).delete() + return n_deleted_items + + logger = logging.getLogger('awx.main.commands.cleanup_deleted') + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(message)s')) + logger.addHandler(handler) + logger.propagate = False + + with transaction.atomic(): + n_deleted_items = 0 + + models = [ + apps.get_model('auth', "User"), + apps.get_model('main', 'Credential'), + apps.get_model('main', 'CustomInventoryScript'), + apps.get_model('main', 'Group'), + apps.get_model('main', 'Host'), + apps.get_model('main', 'Inventory'), + apps.get_model('main', 'Notifier'), + apps.get_model('main', 'Organization'), + apps.get_model('main', 'Permission'), + apps.get_model('main', 'Schedule'), + apps.get_model('main', 'Team'), + apps.get_model('main', 'UnifiedJob'), + apps.get_model('main', 'UnifiedJobTemplate'), + ] + + for model in models: + n_deleted_items += cleanup_model(model) + logger.log(99, "Removed %d items", n_deleted_items)