External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-05-13 20:37:39 -02:30
Code Issues Packages Projects Releases Wiki Activity

Attempt at a workaround for our larger sqlite tests

Browse Source 
These tests are only failing on jenkins, not on our local dev
environments.
This commit is contained in:
Akita Noek
2016-04-18 14:32:21 -04:00
parent 0cf096e5b0
commit 0349737538
1 changed files with 63 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
awx/main/models/rbac.py
View File

@@ -14,6 +14,7 @@ from django.utils.translation import ugettext_lazy as _
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.fields import GenericForeignKey
# AWX # AWX
from django.contrib.auth.models import User # noqa from django.contrib.auth.models import User # noqa
from awx.main.models.base import * # noqa from awx.main.models.base import * # noqa
@@ -58,7 +59,7 @@ def batch_role_ancestor_rebuilding(allow_nesting=False):
if not batch_role_rebuilding: if not batch_role_rebuilding:
rebuild_set = getattr(tls, 'roles_needing_rebuilding') rebuild_set = getattr(tls, 'roles_needing_rebuilding')
with transaction.atomic(): with transaction.atomic():
Role._simultaneous_ancestry_rebuild(rebuild_set) Role._simultaneous_ancestry_rebuild(list(rebuild_set))
#for role in Role.objects.filter(id__in=list(rebuild_set)).all(): #for role in Role.objects.filter(id__in=list(rebuild_set)).all():
# # TODO: We can reduce this to one rebuild call with our new upcoming rebuild method.. do this # # TODO: We can reduce this to one rebuild call with our new upcoming rebuild method.. do this
@@ -128,28 +129,6 @@ class Role(CommonModelNameNotUnique):
roles_needing_rebuilding.add(self.id) roles_needing_rebuilding.add(self.id)
return return
#actual_ancestors = set(Role.objects.filter(id=self.id).values_list('parents__ancestors__id', flat=True))
#actual_ancestors.add(self.id)
#if None in actual_ancestors:
# actual_ancestors.remove(None)
#stored_ancestors = set(self.ancestors.all().values_list('id', flat=True))
'''
# If it differs, update, and then update all of our children
if actual_ancestors != stored_ancestors:
for id in actual_ancestors - stored_ancestors:
self.ancestors.add(id)
for id in stored_ancestors - actual_ancestors:
self.ancestors.remove(id)
for child in self.children.all():
child.rebuild_role_ancestor_list()
'''
# If our role heirarchy hasn't actually changed, don't do anything
#if actual_ancestors == stored_ancestors:
# return
Role._simultaneous_ancestry_rebuild([self.id]) Role._simultaneous_ancestry_rebuild([self.id])
@@ -244,29 +223,19 @@ class Role(CommonModelNameNotUnique):
'ancestors_table': Role.ancestors.through._meta.db_table, 'ancestors_table': Role.ancestors.through._meta.db_table,
'parents_table': Role.parents.through._meta.db_table, 'parents_table': Role.parents.through._meta.db_table,
'roles_table': Role._meta.db_table, 'roles_table': Role._meta.db_table,
'ids': ','.join(str(x) for x in role_ids_to_rebuild)
} }
# This is our solution for dealing with updates to parents of nodes
# that are in cycles. It seems like we should be able to find a more
# clever way of just dealing with the issue in another way, but it's
# surefire and I'm not seeing the easy solution to dealing with that
# problem that's not this.
# TODO: Test to see if not deleting any entry that has a direct def split_ids_for_sqlite(role_ids):
# correponding entry in the parents table helps reduce the processing for i in xrange(0, len(role_ids), 999):
# time significantly yield role_ids[i:i + 999]
"""
cursor.execute(''' for ids in split_ids_for_sqlite(role_ids_to_rebuild):
DELETE FROM %(ancestors_table)s sql_params['ids'] = ','.join(str(x) for x in ids)
WHERE ancestor_id IN (%(ids)s) cursor.execute('''
AND descendent_id != ancestor_id DELETE FROM %(ancestors_table)s
''' % sql_params) WHERE ancestor_id IN (%(ids)s)
""" ''' % sql_params)
cursor.execute('''
DELETE FROM %(ancestors_table)s
WHERE ancestor_id IN (%(ids)s)
''' % sql_params)
while role_ids_to_rebuild: while role_ids_to_rebuild:
@@ -274,66 +243,63 @@ class Role(CommonModelNameNotUnique):
raise Exception('Ancestry role rebuilding error: infinite loop detected') raise Exception('Ancestry role rebuilding error: infinite loop detected')
loop_ct += 1 loop_ct += 1
sql_params = {
'ancestors_table': Role.ancestors.through._meta.db_table,
'parents_table': Role.parents.through._meta.db_table,
'roles_table': Role._meta.db_table,
'ids': ','.join(str(x) for x in role_ids_to_rebuild)
}
delete_ct = 0 delete_ct = 0
for ids in split_ids_for_sqlite(role_ids_to_rebuild):
sql_params['ids'] = ','.join(str(x) for x in ids)
cursor.execute('''
DELETE FROM %(ancestors_table)s
WHERE descendent_id IN (%(ids)s)
AND
id NOT IN (
SELECT %(ancestors_table)s.id FROM (
SELECT parents.from_role_id from_id, ancestors.ancestor_id to_id
FROM %(parents_table)s as parents
LEFT JOIN %(ancestors_table)s as ancestors
ON (parents.to_role_id = ancestors.descendent_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.ancestor_id IS NOT NULL
cursor.execute(''' UNION
DELETE FROM %(ancestors_table)s
WHERE descendent_id IN (%(ids)s)
AND
id NOT IN (
SELECT %(ancestors_table)s.id FROM (
SELECT parents.from_role_id from_id, ancestors.ancestor_id to_id
FROM %(parents_table)s as parents
LEFT JOIN %(ancestors_table)s as ancestors
ON (parents.to_role_id = ancestors.descendent_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.ancestor_id IS NOT NULL
UNION SELECT id from_id, id to_id from %(roles_table)s WHERE id IN (%(ids)s)
) new_ancestry_list
LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.descendent_id
AND new_ancestry_list.to_id = %(ancestors_table)s.ancestor_id)
WHERE %(ancestors_table)s.id IS NOT NULL
)
''' % sql_params)
delete_ct += cursor.rowcount
SELECT id from_id, id to_id from %(roles_table)s WHERE id IN (%(ids)s) insert_ct = 0
) new_ancestry_list for ids in split_ids_for_sqlite(role_ids_to_rebuild):
LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.descendent_id sql_params['ids'] = ','.join(str(x) for x in ids)
AND new_ancestry_list.to_id = %(ancestors_table)s.ancestor_id) cursor.execute('''
WHERE %(ancestors_table)s.id IS NOT NULL INSERT INTO %(ancestors_table)s (descendent_id, ancestor_id, role_field, content_type_id, object_id)
) SELECT from_id, to_id, new_ancestry_list.role_field, new_ancestry_list.content_type_id, new_ancestry_list.object_id FROM (
''' % sql_params) SELECT parents.from_role_id from_id,
delete_ct = cursor.rowcount ancestors.ancestor_id to_id,
roles.role_field,
COALESCE(roles.content_type_id, 0) content_type_id,
COALESCE(roles.object_id, 0) object_id
FROM %(parents_table)s as parents
INNER JOIN %(roles_table)s as roles ON (parents.from_role_id = roles.id)
LEFT OUTER JOIN %(ancestors_table)s as ancestors
ON (parents.to_role_id = ancestors.descendent_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.ancestor_id IS NOT NULL
cursor.execute(''' UNION
INSERT INTO %(ancestors_table)s (descendent_id, ancestor_id, role_field, content_type_id, object_id)
SELECT from_id, to_id, new_ancestry_list.role_field, new_ancestry_list.content_type_id, new_ancestry_list.object_id FROM (
SELECT parents.from_role_id from_id,
ancestors.ancestor_id to_id,
roles.role_field,
COALESCE(roles.content_type_id, 0) content_type_id,
COALESCE(roles.object_id, 0) object_id
FROM %(parents_table)s as parents
INNER JOIN %(roles_table)s as roles ON (parents.from_role_id = roles.id)
LEFT OUTER JOIN %(ancestors_table)s as ancestors
ON (parents.to_role_id = ancestors.descendent_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.ancestor_id IS NOT NULL
UNION SELECT id from_id,
id to_id,
SELECT id from_id, role_field,
id to_id, COALESCE(content_type_id, 0) content_type_id,
role_field, COALESCE(object_id, 0) object_id
COALESCE(content_type_id, 0) content_type_id, from %(roles_table)s WHERE id IN (%(ids)s)
COALESCE(object_id, 0) object_id ) new_ancestry_list
from %(roles_table)s WHERE id IN (%(ids)s) LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.descendent_id
) new_ancestry_list AND new_ancestry_list.to_id = %(ancestors_table)s.ancestor_id)
LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.descendent_id WHERE %(ancestors_table)s.id IS NULL
AND new_ancestry_list.to_id = %(ancestors_table)s.ancestor_id) ''' % sql_params)
WHERE %(ancestors_table)s.id IS NULL insert_ct += cursor.rowcount
''' % sql_params)
insert_ct = cursor.rowcount
if insert_ct == 0 and delete_ct == 0: if insert_ct == 0 and delete_ct == 0:
break break