External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-23 03:45:01 -02:30
Code Issues Packages Projects Releases Wiki Activity

Futher optimze role rebuilding to be aware of whether we are adding or removing parentage

Browse Source
This commit is contained in:
Akita Noek
2016-04-21 17:05:00 -04:00
parent 569f61ed30
commit 17120ffe4f
3 changed files with 93 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
awx/main/fields.py
View File

@@ -200,7 +200,7 @@ class ImplicitRoleField(models.ForeignKey):
updates[role.role_field] = role.id updates[role.role_field] = role.id
role_ids.append(role.id) role_ids.append(role.id)
type(instance).objects.filter(pk=instance.pk).update(**updates) type(instance).objects.filter(pk=instance.pk).update(**updates)
Role_._simultaneous_ancestry_rebuild(role_ids) Role_.rebuild_role_ancestor_list(role_ids, [])
# Update parentage if necessary # Update parentage if necessary
for implicit_role_field in getattr(instance.__class__, '__implicit_role_fields'): for implicit_role_field in getattr(instance.__class__, '__implicit_role_fields'):
@@ -256,4 +256,4 @@ class ImplicitRoleField(models.ForeignKey):
Role_ = get_current_apps().get_model('main', 'Role') Role_ = get_current_apps().get_model('main', 'Role')
child_ids = [x for x in Role_.parents.through.objects.filter(to_role_id__in=role_ids).distinct().values_list('from_role_id', flat=True)] child_ids = [x for x in Role_.parents.through.objects.filter(to_role_id__in=role_ids).distinct().values_list('from_role_id', flat=True)]
Role_.objects.filter(id__in=role_ids).delete() Role_.objects.filter(id__in=role_ids).delete()
Role_._simultaneous_ancestry_rebuild(child_ids) Role_.rebuild_role_ancestor_list([], child_ids)

159
awx/main/models/rbac.py
View File

@@ -82,20 +82,19 @@ def batch_role_ancestor_rebuilding(allow_nesting=False):
try: try:
setattr(tls, 'batch_role_rebuilding', True) setattr(tls, 'batch_role_rebuilding', True)
if not batch_role_rebuilding: if not batch_role_rebuilding:
setattr(tls, 'roles_needing_rebuilding', set()) setattr(tls, 'additions', set())
setattr(tls, 'removals', set())
yield yield
finally: finally:
setattr(tls, 'batch_role_rebuilding', batch_role_rebuilding) setattr(tls, 'batch_role_rebuilding', batch_role_rebuilding)
if not batch_role_rebuilding: if not batch_role_rebuilding:
rebuild_set = getattr(tls, 'roles_needing_rebuilding') additions = getattr(tls, 'additions')
removals = getattr(tls, 'removals')
with transaction.atomic(): with transaction.atomic():
Role._simultaneous_ancestry_rebuild(list(rebuild_set)) Role.rebuild_role_ancestor_list(list(additions), list(removals))
delattr(tls, 'additions')
#for role in Role.objects.filter(id__in=list(rebuild_set)).all(): delattr(tls, 'removals')
# # TODO: We can reduce this to one rebuild call with our new upcoming rebuild method.. do this
# role.rebuild_role_ancestor_list()
delattr(tls, 'roles_needing_rebuilding')
class Role(models.Model): class Role(models.Model):
@@ -125,7 +124,7 @@ class Role(models.Model):
def save(self, *args, **kwargs): def save(self, *args, **kwargs):
super(Role, self).save(*args, **kwargs) super(Role, self).save(*args, **kwargs)
self.rebuild_role_ancestor_list() self.rebuild_role_ancestor_list([self.id], [])
def get_absolute_url(self): def get_absolute_url(self):
return reverse('api:role_detail', args=(self.pk,)) return reverse('api:role_detail', args=(self.pk,))
@@ -143,17 +142,6 @@ class Role(models.Model):
object_id=accessor.id) object_id=accessor.id)
return self.ancestors.filter(pk__in=roles).exists() return self.ancestors.filter(pk__in=roles).exists()
def rebuild_role_ancestor_list(self):
'''
Updates our `ancestors` map to accurately reflect all of the ancestors for a role
You should never need to call this. Signal handlers should be calling
this method when the role hierachy changes automatically.
Note that this method relies on any parents' ancestor list being correct.
'''
Role._simultaneous_ancestry_rebuild([self.id])
@property @property
def name(self): def name(self):
global role_names global role_names
@@ -165,7 +153,13 @@ class Role(models.Model):
return role_descriptions[self.role_field] return role_descriptions[self.role_field]
@staticmethod @staticmethod
def _simultaneous_ancestry_rebuild(role_ids_to_rebuild): def rebuild_role_ancestor_list(additions, removals):
'''
Updates our `ancestors` map to accurately reflect all of the ancestors for a role
You should never need to call this. Signal handlers should be calling
this method when the role hierachy changes automatically.
'''
# The ancestry table # The ancestry table
# ================================================= # =================================================
# #
@@ -238,15 +232,15 @@ class Role(models.Model):
# #
# #
if len(role_ids_to_rebuild) == 0: if len(additions) == 0 and len(removals) == 0:
return return
global tls global tls
batch_role_rebuilding = getattr(tls, 'batch_role_rebuilding', False) batch_role_rebuilding = getattr(tls, 'batch_role_rebuilding', False)
if batch_role_rebuilding: if batch_role_rebuilding:
roles_needing_rebuilding = getattr(tls, 'roles_needing_rebuilding') getattr(tls, 'additions').update(set(additions))
roles_needing_rebuilding.update(set(role_ids_to_rebuild)) getattr(tls, 'removals').update(set(removals))
return return
cursor = connection.cursor() cursor = connection.cursor()
@@ -268,77 +262,88 @@ class Role(models.Model):
for i in xrange(0, len(role_ids), 40000): for i in xrange(0, len(role_ids), 40000):
yield role_ids[i:i + 40000] yield role_ids[i:i + 40000]
with transaction.atomic(): with transaction.atomic():
while role_ids_to_rebuild: while len(additions) > 0 or len(removals) > 0:
if loop_ct > 100: if loop_ct > 100:
raise Exception('Role ancestry rebuilding error: infinite loop detected') raise Exception('Role ancestry rebuilding error: infinite loop detected')
loop_ct += 1 loop_ct += 1
delete_ct = 0 delete_ct = 0
for ids in split_ids_for_sqlite(role_ids_to_rebuild): if len(removals) > 0:
sql_params['ids'] = ','.join(str(x) for x in ids) for ids in split_ids_for_sqlite(removals):
cursor.execute(''' sql_params['ids'] = ','.join(str(x) for x in ids)
DELETE FROM %(ancestors_table)s cursor.execute('''
WHERE descendent_id IN (%(ids)s) DELETE FROM %(ancestors_table)s
AND descendent_id != ancestor_id WHERE descendent_id IN (%(ids)s)
AND NOT EXISTS ( AND descendent_id != ancestor_id
SELECT 1 AND NOT EXISTS (
FROM %(parents_table)s as parents SELECT 1
INNER JOIN %(ancestors_table)s as inner_ancestors FROM %(parents_table)s as parents
ON (parents.to_role_id = inner_ancestors.descendent_id) INNER JOIN %(ancestors_table)s as inner_ancestors
WHERE parents.from_role_id = %(ancestors_table)s.descendent_id ON (parents.to_role_id = inner_ancestors.descendent_id)
AND %(ancestors_table)s.ancestor_id = inner_ancestors.ancestor_id WHERE parents.from_role_id = %(ancestors_table)s.descendent_id
) AND %(ancestors_table)s.ancestor_id = inner_ancestors.ancestor_id
''' % sql_params) )
''' % sql_params)
delete_ct += cursor.rowcount delete_ct += cursor.rowcount
insert_ct = 0 insert_ct = 0
for ids in split_ids_for_sqlite(role_ids_to_rebuild): if len(additions) > 0:
sql_params['ids'] = ','.join(str(x) for x in ids) for ids in split_ids_for_sqlite(additions):
cursor.execute(''' sql_params['ids'] = ','.join(str(x) for x in ids)
INSERT INTO %(ancestors_table)s (descendent_id, ancestor_id, role_field, content_type_id, object_id) cursor.execute('''
SELECT from_id, to_id, new_ancestry_list.role_field, new_ancestry_list.content_type_id, new_ancestry_list.object_id FROM ( INSERT INTO %(ancestors_table)s (descendent_id, ancestor_id, role_field, content_type_id, object_id)
SELECT roles.id from_id, SELECT from_id, to_id, new_ancestry_list.role_field, new_ancestry_list.content_type_id, new_ancestry_list.object_id FROM (
ancestors.ancestor_id to_id, SELECT roles.id from_id,
roles.role_field, ancestors.ancestor_id to_id,
COALESCE(roles.content_type_id, 0) content_type_id, roles.role_field,
COALESCE(roles.object_id, 0) object_id COALESCE(roles.content_type_id, 0) content_type_id,
FROM %(roles_table)s as roles COALESCE(roles.object_id, 0) object_id
INNER JOIN %(parents_table)s as parents FROM %(roles_table)s as roles
ON (parents.from_role_id = roles.id) INNER JOIN %(parents_table)s as parents
INNER JOIN %(ancestors_table)s as ancestors ON (parents.from_role_id = roles.id)
ON (parents.to_role_id = ancestors.descendent_id) INNER JOIN %(ancestors_table)s as ancestors
WHERE roles.id IN (%(ids)s) ON (parents.to_role_id = ancestors.descendent_id)
WHERE roles.id IN (%(ids)s)
UNION UNION
SELECT id from_id, SELECT id from_id,
id to_id, id to_id,
role_field, role_field,
COALESCE(content_type_id, 0) content_type_id, COALESCE(content_type_id, 0) content_type_id,
COALESCE(object_id, 0) object_id COALESCE(object_id, 0) object_id
from %(roles_table)s WHERE id IN (%(ids)s) from %(roles_table)s WHERE id IN (%(ids)s)
) new_ancestry_list ) new_ancestry_list
WHERE NOT EXISTS ( WHERE NOT EXISTS (
SELECT 1 FROM %(ancestors_table)s SELECT 1 FROM %(ancestors_table)s
WHERE %(ancestors_table)s.descendent_id = new_ancestry_list.from_id WHERE %(ancestors_table)s.descendent_id = new_ancestry_list.from_id
AND %(ancestors_table)s.ancestor_id = new_ancestry_list.to_id AND %(ancestors_table)s.ancestor_id = new_ancestry_list.to_id
) )
''' % sql_params) ''' % sql_params)
insert_ct += cursor.rowcount insert_ct += cursor.rowcount
if insert_ct == 0 and delete_ct == 0: if insert_ct == 0 and delete_ct == 0:
break break
new_role_ids_to_rebuild = set() new_additions = set()
for ids in split_ids_for_sqlite(role_ids_to_rebuild): for ids in split_ids_for_sqlite(additions):
sql_params['ids'] = ','.join(str(x) for x in ids) sql_params['ids'] = ','.join(str(x) for x in ids)
# get all children for the roles we're operating on # get all children for the roles we're operating on
cursor.execute('SELECT DISTINCT from_role_id FROM %(parents_table)s WHERE to_role_id IN (%(ids)s)' % sql_params) cursor.execute('SELECT DISTINCT from_role_id FROM %(parents_table)s WHERE to_role_id IN (%(ids)s)' % sql_params)
new_role_ids_to_rebuild.update([row[0] for row in cursor.fetchall()]) new_additions.update([row[0] for row in cursor.fetchall()])
role_ids_to_rebuild = list(new_role_ids_to_rebuild) additions = list(new_additions)
new_removals = set()
for ids in split_ids_for_sqlite(removals):
sql_params['ids'] = ','.join(str(x) for x in ids)
# get all children for the roles we're operating on
cursor.execute('SELECT DISTINCT from_role_id FROM %(parents_table)s WHERE to_role_id IN (%(ids)s)' % sql_params)
new_removals.update([row[0] for row in cursor.fetchall()])
removals = list(new_removals)
@staticmethod @staticmethod
@@ -362,7 +367,7 @@ class RoleAncestorEntry(models.Model):
index_together = [ index_together = [
("ancestor", "content_type_id", "object_id"), # used by get_roles_on_resource ("ancestor", "content_type_id", "object_id"), # used by get_roles_on_resource
("ancestor", "content_type_id", "role_field"), # used by accessible_objects ("ancestor", "content_type_id", "role_field"), # used by accessible_objects
("ancestor", "descendent"), # used by _simultaneous_ancestry_rebuild in the NOT EXISTS clauses. ("ancestor", "descendent"), # used by rebuild_role_ancestor_list in the NOT EXISTS clauses.
] ]
descendent = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+') descendent = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')

13
awx/main/signals.py
View File

@@ -108,12 +108,17 @@ def emit_update_inventory_on_created_or_deleted(sender, **kwargs):
def rebuild_role_ancestor_list(reverse, model, instance, pk_set, action, **kwargs): def rebuild_role_ancestor_list(reverse, model, instance, pk_set, action, **kwargs):
'When a role parent is added or removed, update our role hierarchy list' 'When a role parent is added or removed, update our role hierarchy list'
if action in ['post_add', 'post_remove', 'post_clear']: if action == 'post_add':
if reverse: if reverse:
for id in pk_set: model.rebuild_role_ancestor_list(list(pk_set), [])
model.objects.get(id=id).rebuild_role_ancestor_list()
else: else:
instance.rebuild_role_ancestor_list() model.rebuild_role_ancestor_list([instance.id], [])
if action in ['post_remove', 'post_clear']:
if reverse:
model.rebuild_role_ancestor_list([], list(pk_set))
else:
model.rebuild_role_ancestor_list([], [instance.id])
def sync_superuser_status_to_rbac(instance, **kwargs): def sync_superuser_status_to_rbac(instance, **kwargs):
'When the is_superuser flag is changed on a user, reflect that in the membership of the System Admnistrator role' 'When the is_superuser flag is changed on a user, reflect that in the membership of the System Admnistrator role'