Attempt at a workaround for our larger sqlite tests

These tests are only failing on jenkins, not on our local dev
environments.

awx/main/models/rbac.py

@@ -14,6 +14,7 @@ from django.utils.translation import ugettext_lazy as _
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.contenttypes.fields import GenericForeignKey
 
+
 # AWX
 from django.contrib.auth.models import User # noqa
 from awx.main.models.base import * # noqa
@@ -58,7 +59,7 @@ def batch_role_ancestor_rebuilding(allow_nesting=False):
         if not batch_role_rebuilding:
             rebuild_set = getattr(tls, 'roles_needing_rebuilding')
             with transaction.atomic():
-                Role._simultaneous_ancestry_rebuild(rebuild_set)
+                Role._simultaneous_ancestry_rebuild(list(rebuild_set))
 
                 #for role in Role.objects.filter(id__in=list(rebuild_set)).all():
                 #    # TODO: We can reduce this to one rebuild call with our new upcoming rebuild method.. do this
@@ -128,28 +129,6 @@ class Role(CommonModelNameNotUnique):
             roles_needing_rebuilding.add(self.id)
             return
 
-        #actual_ancestors = set(Role.objects.filter(id=self.id).values_list('parents__ancestors__id', flat=True))
-        #actual_ancestors.add(self.id)
-        #if None in actual_ancestors:
-        #    actual_ancestors.remove(None)
-        #stored_ancestors = set(self.ancestors.all().values_list('id', flat=True))
-
-        '''
-        # If it differs, update, and then update all of our children
-        if actual_ancestors != stored_ancestors:
-            for id in actual_ancestors - stored_ancestors:
-                self.ancestors.add(id)
-            for id in stored_ancestors - actual_ancestors:
-                self.ancestors.remove(id)
-
-            for child in self.children.all():
-                child.rebuild_role_ancestor_list()
-        '''
-
-        # If our role heirarchy hasn't actually changed, don't do anything
-        #if actual_ancestors == stored_ancestors:
-        #    return
-
         Role._simultaneous_ancestry_rebuild([self.id])
 
 
@@ -244,25 +223,15 @@ class Role(CommonModelNameNotUnique):
             'ancestors_table': Role.ancestors.through._meta.db_table,
             'parents_table': Role.parents.through._meta.db_table,
             'roles_table': Role._meta.db_table,
-            'ids': ','.join(str(x) for x in role_ids_to_rebuild)
         }
 
-        # This is our solution for dealing with updates to parents of nodes
-        # that are in cycles. It seems like we should be able to find a more
-        # clever way of just dealing with the issue in another way, but it's
-        # surefire and I'm not seeing the easy solution to dealing with that
-        # problem that's not this.
 
-        # TODO: Test to see if not deleting any entry that has a direct
+        def split_ids_for_sqlite(role_ids):
-        # correponding entry in the parents table helps reduce the processing
+            for i in xrange(0, len(role_ids), 999):
-        # time significantly
+                yield role_ids[i:i + 999]
-        """
+
-        cursor.execute('''
+        for ids in split_ids_for_sqlite(role_ids_to_rebuild):
-            DELETE FROM %(ancestors_table)s
+            sql_params['ids'] = ','.join(str(x) for x in ids)
-              WHERE ancestor_id IN (%(ids)s)
-                    AND descendent_id != ancestor_id
-        ''' % sql_params)
-        """
             cursor.execute('''
                  DELETE FROM %(ancestors_table)s
                   WHERE ancestor_id IN (%(ids)s)
@@ -274,15 +243,9 @@ class Role(CommonModelNameNotUnique):
                 raise Exception('Ancestry role rebuilding error: infinite loop detected')
             loop_ct += 1
 
-            sql_params = {
-                'ancestors_table': Role.ancestors.through._meta.db_table,
-                'parents_table': Role.parents.through._meta.db_table,
-                'roles_table': Role._meta.db_table,
-                'ids': ','.join(str(x) for x in role_ids_to_rebuild)
-            }
-
             delete_ct = 0
-
+            for ids in split_ids_for_sqlite(role_ids_to_rebuild):
+                sql_params['ids'] = ','.join(str(x) for x in ids)
                 cursor.execute('''
                     DELETE FROM %(ancestors_table)s
                     WHERE descendent_id IN (%(ids)s)
@@ -304,8 +267,11 @@ class Role(CommonModelNameNotUnique):
                                WHERE %(ancestors_table)s.id IS NOT NULL
                          )
                 ''' % sql_params)
-            delete_ct = cursor.rowcount
+                delete_ct += cursor.rowcount
 
+            insert_ct = 0
+            for ids in split_ids_for_sqlite(role_ids_to_rebuild):
+                sql_params['ids'] = ','.join(str(x) for x in ids)
                 cursor.execute('''
                     INSERT INTO %(ancestors_table)s (descendent_id, ancestor_id, role_field, content_type_id, object_id)
                     SELECT from_id, to_id, new_ancestry_list.role_field, new_ancestry_list.content_type_id, new_ancestry_list.object_id FROM  (
@@ -333,7 +299,7 @@ class Role(CommonModelNameNotUnique):
                                                              AND new_ancestry_list.to_id = %(ancestors_table)s.ancestor_id)
                      WHERE %(ancestors_table)s.id IS NULL
                 ''' % sql_params)
-            insert_ct = cursor.rowcount
+                insert_ct += cursor.rowcount
 
             if insert_ct == 0 and delete_ct == 0:
                 break