First pass at cleanup_deleted management command.

awx/main/base_views.py

@@ -1,23 +1,28 @@
 # Copyright (c) 2013 AnsibleWorks, Inc.
 # All Rights Reserved.
 
+# Python
+import json
 
+# Django
 from django.http import HttpResponse, Http404
 from django.views.decorators.csrf import csrf_exempt
-from awx.main.models import *
 from django.contrib.auth.models import User
-from awx.main.serializers import *
+from django.utils.timezone import now
-from awx.main.rbac import *
+
-from awx.main.access import *
+# Django REST Framework
 from rest_framework.exceptions import PermissionDenied
 from rest_framework import mixins
 from rest_framework import generics
 from rest_framework import permissions
 from rest_framework.response import Response
 from rest_framework import status
-import exceptions
+
-import datetime
+# AWX
-import json as python_json
+from awx.main.models import *
+from awx.main.serializers import *
+from awx.main.rbac import *
+from awx.main.access import *
 
 # FIXME: machinery for auto-adding audit trail logs to all CREATE/EDITS
 
@@ -128,7 +133,7 @@ class BaseSubList(BaseList):
                                     if not organization.admins.filter(pk=request.user.pk).count() > 0:
                                         raise PermissionDenied()
                             else:
-                                raise exceptions.NotImplementedError()
+                                raise NotImplementedError()
                         else:
                             if not check_user_access(request.user, type(obj), 'read', obj):
                                 raise PermissionDenied()
@@ -162,7 +167,7 @@ class BaseSubList(BaseList):
                 else:
 
                     # view didn't specify a way to get the pk from the URL, so not even trying
-                    return Response(status=status.HTTP_400_BAD_REQUEST, data=python_json.dumps(dict(msg='object cannot be created')))
+                    return Response(status=status.HTTP_400_BAD_REQUEST, data=json.dumps(dict(msg='object cannot be created')))
 
         # we didn't have to create the object, so this is just associating the two objects together now...
         # (or disassociating them)
@@ -229,7 +234,7 @@ class BaseDetail(generics.RetrieveUpdateDestroyAPIView):
         if isinstance(obj, PrimordialModel):
             obj.mark_inactive()
         elif type(obj) == User:
-            obj.username  = "_deleted_%s_%s" % (str(datetime.time()), obj.username)
+            obj.username  = "_deleted_%s_%s" % (now().isoformat(), obj.username)
             obj.is_active = False
             obj.save()
         else:

awx/main/management/commands/cleanup_deleted.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2013 AnsibleWorks, Inc.
+# All Rights Reserved.
+
+# Python
+import datetime
+import logging
+from optparse import make_option
+
+# Django
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+from django.contrib.auth.models import User
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import now
+
+# AWX
+from awx.main.models import *
+
+class Command(BaseCommand):
+    '''
+    Management command to cleanup deleted items.
+    '''
+
+    help = 'Cleanup deleted items from the database.'
+    args = '[<appname>, <appname.ModelName>, ...]'
+
+    option_list = BaseCommand.option_list + (
+        make_option('--days', dest='days', type='int', default=90, metavar='N',
+                    help='Remove items deleted more than N days ago'),
+        make_option('--dry-run', dest='dry_run', action='store_true',
+                    default=False, help='Dry run mode (show items that would '
+                    'be removed)'),
+    )
+
+    def get_models(self, model):
+        if not model._meta.abstract:
+            yield model
+        for sub in model.__subclasses__():
+            for submodel in self.get_models(sub):
+                yield submodel
+
+    def cleanup_model(self, model):
+        name_field = None
+        active_field = None
+        for field in model._meta.fields:
+            if field.name in ('name', 'username'):
+                name_field = field.name
+            if field.name in ('is_active', 'active'):
+                active_field = field.name
+        if not name_field:
+            self.logger.warning('skipping model %s, no name field', model)
+            return
+        if not active_field:
+            self.logger.warning('skipping model %s, no active field', model)
+            return
+        qs = model.objects.filter(**{
+            active_field: False,
+            '%s__startswith' % name_field: '_deleted_',
+        })
+        self.logger.debug('cleaning up model %s', model)
+        for instance in qs:
+            dt = parse_datetime(getattr(instance, name_field).split('_')[2])
+            if not dt:
+                self.logger.warning('unable to find deleted timestamp in %s '
+                                    'field', name_field)
+            elif dt >= self.cutoff:
+                action_text = 'would skip' if self.dry_run else 'skipping'
+                self.logger.debug('%s %s', action_text, instance)
+            else:
+                action_text = 'would delete' if self.dry_run else 'deleting'
+                self.logger.info('%s %s', action_text, instance)
+                if not self.dry_run:
+                    instance.delete()
+
+    def init_logging(self):
+        log_levels = dict(enumerate([logging.ERROR, logging.INFO,
+                                     logging.DEBUG, 0]))
+        self.logger = logging.getLogger('awx.main.commands.cleanup_deleted')
+        self.logger.setLevel(log_levels.get(self.verbosity, 0))
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter('%(message)s'))
+        self.logger.addHandler(handler)
+        self.logger.propagate = False
+
+    @transaction.commit_on_success
+    def handle(self, *args, **options):
+        self.verbosity = int(options.get('verbosity', 1))
+        self.init_logging()
+        self.days = int(options.get('days', 90))
+        self.dry_run = bool(options.get('dry_run', False))
+        # FIXME: Handle args to select models.
+        self.cutoff = now() - datetime.timedelta(days=self.days)
+        self.cleanup_model(User)
+        for model in self.get_models(PrimordialModel):
+            self.cleanup_model(model)

awx/main/tests/__init__.py

@@ -5,6 +5,7 @@ from awx.main.tests.organizations import OrganizationsTest
 from awx.main.tests.users import UsersTest
 from awx.main.tests.inventory import InventoryTest
 from awx.main.tests.projects import ProjectsTest
+from awx.main.tests.commands import *
 from awx.main.tests.scripts import *
 from awx.main.tests.tasks import RunJobTest
 from awx.main.tests.licenses import LicenseTests

awx/main/tests/commands.py

@@ -0,0 +1,207 @@
+# Copyright (c) 2013 AnsibleWorks, Inc.
+# All Rights Reserved.
+
+# Python
+import json
+import os
+import StringIO
+import sys
+import tempfile
+
+# Django
+from django.conf import settings
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.utils.timezone import now
+
+# AWX
+from awx.main.models import *
+from awx.main.tests.base import BaseTest
+
+__all__ = ['CleanupDeletedTest']
+
+class BaseCommandTest(BaseTest):
+    '''
+    Base class for tests that run management commands.
+    '''
+
+    def setUp(self):
+        super(BaseCommandTest, self).setUp()
+        self._sys_path = [x for x in sys.path]
+        self._environ = dict(os.environ.items())
+        self._temp_files = []
+
+    def tearDown(self):
+        super(BaseCommandTest, self).tearDown()
+        sys.path = self._sys_path
+        for k,v in self._environ.items():
+            if os.environ.get(k, None) != v:
+                os.environ[k] = v
+        for k,v in os.environ.items():
+            if k not in self._environ.keys():
+                del os.environ[k]
+        for tf in self._temp_files:
+            if os.path.exists(tf):
+                os.remove(tf)
+
+    def run_command(self, name, *args, **options):
+        '''
+        Run a management command and capture its stdout/stderr along with any
+        exceptions.
+        '''
+        command_runner = options.pop('command_runner', call_command)
+        stdin_fileobj = options.pop('stdin_fileobj', None)
+        options.setdefault('verbosity', 1)
+        options.setdefault('interactive', False)
+        original_stdin = sys.stdin
+        original_stdout = sys.stdout
+        original_stderr = sys.stderr
+        if stdin_fileobj:
+            sys.stdin = stdin_fileobj
+        sys.stdout = StringIO.StringIO()
+        sys.stderr = StringIO.StringIO()
+        result = None
+        try:
+            result = command_runner(name, *args, **options)
+        except Exception, e:
+            result = e
+        except SystemExit, e:
+            result = e
+        finally:
+            captured_stdout = sys.stdout.getvalue()
+            captured_stderr = sys.stderr.getvalue()
+            sys.stdin = original_stdin
+            sys.stdout = original_stdout
+            sys.stderr = original_stderr
+        return result, captured_stdout, captured_stderr
+
+class CleanupDeletedTest(BaseCommandTest):
+    '''
+    Test cases for cleanup_deleted management command.
+    '''
+
+    def setUp(self):
+        super(CleanupDeletedTest, self).setUp()
+        self.setup_users()
+        self.organizations = self.make_organizations(self.super_django_user, 2)
+        self.projects = self.make_projects(self.normal_django_user, 2)
+        self.organizations[0].projects.add(self.projects[1])
+        self.organizations[1].projects.add(self.projects[0])
+        self.inventories = []
+        self.hosts = []
+        self.groups = []
+        for n, organization in enumerate(self.organizations):
+            inventory = Inventory.objects.create(name='inventory-%d' % n,
+                                                 description='description for inventory %d' % n,
+                                                 organization=organization,
+                                                 variables=json.dumps({'n': n}) if n else '')
+            self.inventories.append(inventory)
+            hosts = []
+            for x in xrange(10):
+                if n > 0:
+                    variables = json.dumps({'ho': 'hum-%d' % x})
+                else:
+                    variables = ''
+                host = inventory.hosts.create(name='host-%02d-%02d.example.com' % (n, x),
+                                              inventory=inventory,
+                                              variables=variables)
+                hosts.append(host)
+            self.hosts.extend(hosts)
+            groups = []
+            for x in xrange(5):
+                if n > 0:
+                    variables = json.dumps({'gee': 'whiz-%d' % x})
+                else:
+                    variables = ''
+                group = inventory.groups.create(name='group-%d' % x,
+                                                inventory=inventory,
+                                                variables=variables)
+                groups.append(group)
+                group.hosts.add(hosts[x])
+                group.hosts.add(hosts[x + 5])
+                if n > 0 and x == 4:
+                    group.parents.add(groups[3])
+            self.groups.extend(groups)
+
+    def get_model_counts(self):
+        def get_models(m):
+            if not m._meta.abstract:
+                yield m
+            for sub in m.__subclasses__():
+                for subm in get_models(sub):
+                    yield subm
+        counts = {}
+        for model in get_models(PrimordialModel):
+            active = model.objects.filter(active=True).count()
+            inactive = model.objects.filter(active=False).count()
+            counts[model] = (active, inactive)
+        return counts
+
+    def test_cleanup_our_models(self):
+        # Test with nothing to be deleted.
+        counts_before = self.get_model_counts()
+        self.assertFalse(sum(x[1] for x in counts_before.values()))
+        result, stdout, stderr = self.run_command('cleanup_deleted')
+        self.assertEqual(result, None)
+        counts_after = self.get_model_counts()
+        self.assertEqual(counts_before, counts_after)
+        # "Delete" some hosts.
+        for host in Host.objects.all():
+            host.mark_inactive()
+        # With no parameters, "days" defaults to 90, which won't cleanup any of
+        # the hosts we just removed.
+        counts_before = self.get_model_counts()
+        self.assertTrue(sum(x[1] for x in counts_before.values()))
+        result, stdout, stderr = self.run_command('cleanup_deleted')
+        self.assertEqual(result, None)
+        counts_after = self.get_model_counts()
+        self.assertEqual(counts_before, counts_after)
+        # Even with days=1, the hosts will remain.
+        counts_before = self.get_model_counts()
+        self.assertTrue(sum(x[1] for x in counts_before.values()))
+        result, stdout, stderr = self.run_command('cleanup_deleted', days=1)
+        self.assertEqual(result, None)
+        counts_after = self.get_model_counts()
+        self.assertEqual(counts_before, counts_after)
+        # With days=0, the hosts will be deleted.
+        counts_before = self.get_model_counts()
+        self.assertTrue(sum(x[1] for x in counts_before.values()))
+        result, stdout, stderr = self.run_command('cleanup_deleted', days=0)
+        self.assertEqual(result, None)
+        counts_after = self.get_model_counts()
+        self.assertNotEqual(counts_before, counts_after)
+        self.assertFalse(sum(x[1] for x in counts_after.values()))
+
+    def get_user_counts(self):
+        active = User.objects.filter(is_active=True).count()
+        inactive = User.objects.filter(is_active=False).count()
+        return active, inactive
+
+    def test_cleanup_user_model(self):
+        # Test with nothing to be deleted.
+        counts_before = self.get_user_counts()
+        self.assertFalse(counts_before[1])
+        result, stdout, stderr = self.run_command('cleanup_deleted')
+        self.assertEqual(result, None)
+        counts_after = self.get_user_counts()
+        self.assertEqual(counts_before, counts_after)
+        # "Delete some users".
+        for user in User.objects.all():
+            user.username  = "_deleted_%s_%s" % (now().isoformat(), user.username)
+            user.is_active = False
+            user.save()
+        # With days=1, no users will be deleted.
+        counts_before = self.get_user_counts()
+        self.assertTrue(counts_before[1])
+        result, stdout, stderr = self.run_command('cleanup_deleted', days=1)
+        self.assertEqual(result, None)
+        counts_after = self.get_user_counts()
+        self.assertEqual(counts_before, counts_after)
+        # With days=0, inactive users will be deleted.
+        counts_before = self.get_user_counts()
+        self.assertTrue(counts_before[1])
+        result, stdout, stderr = self.run_command('cleanup_deleted', days=0)
+        self.assertEqual(result, None)
+        counts_after = self.get_user_counts()
+        self.assertNotEqual(counts_before, counts_after)
+        self.assertFalse(counts_after[1])