External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-22 03:17:39 -02:30
Code Issues Packages Projects Releases Wiki Activity

Switch to custom ancestry table for some optimized queries

Browse Source 
Now we can stuff some more data in this table so we can take advantage
of some multi-column indexing and avoid another to join for our
accessible objects and permissions queries.
This commit is contained in:
Akita Noek
2016-04-16 18:27:57 -04:00
parent 25303cf4ec
commit 5d0c6cc044
3 changed files with 109 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
awx/main/migrations/0008_v300_rbac_changes.py
View File

@@ -48,6 +48,19 @@ class Migration(migrations.Migration):
field=models.ManyToManyField(related_name='deprecated_teams', to='main.Project', blank=True), field=models.ManyToManyField(related_name='deprecated_teams', to='main.Project', blank=True),
), ),
migrations.CreateModel(
name='RoleAncestorEntry',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('role_field', models.TextField()),
('content_type_id', models.PositiveIntegerField(null=False)),
('object_id', models.PositiveIntegerField(null=False)),
],
options={
'db_table': 'main_rbac_role_ancestors',
'verbose_name_plural': 'role_ancestors',
},
),
migrations.CreateModel( migrations.CreateModel(
name='Role', name='Role',
fields=[ fields=[
@@ -58,7 +71,7 @@ class Migration(migrations.Migration):
('name', models.CharField(max_length=512)), ('name', models.CharField(max_length=512)),
('singleton_name', models.TextField(default=None, unique=True, null=True, db_index=True)), ('singleton_name', models.TextField(default=None, unique=True, null=True, db_index=True)),
('object_id', models.PositiveIntegerField(default=None, null=True)), ('object_id', models.PositiveIntegerField(default=None, null=True)),
('ancestors', models.ManyToManyField(related_name='descendents', to='main.Role')), ('ancestors', models.ManyToManyField(related_name='descendents', through='main.RoleAncestorEntry', to='main.Role')),
('content_type', models.ForeignKey(default=None, to='contenttypes.ContentType', null=True)), ('content_type', models.ForeignKey(default=None, to='contenttypes.ContentType', null=True)),
('created_by', models.ForeignKey(related_name="{u'class': 'role', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)), ('created_by', models.ForeignKey(related_name="{u'class': 'role', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
('members', models.ManyToManyField(related_name='roles', to=settings.AUTH_USER_MODEL)), ('members', models.ManyToManyField(related_name='roles', to=settings.AUTH_USER_MODEL)),
@@ -72,6 +85,20 @@ class Migration(migrations.Migration):
'verbose_name_plural': 'roles', 'verbose_name_plural': 'roles',
}, },
), ),
migrations.AddField(
model_name='roleancestorentry',
name='ancestor',
field=models.ForeignKey(related_name='+', to='main.Role'),
),
migrations.AddField(
model_name='roleancestorentry',
name='descendent',
field=models.ForeignKey(related_name='+', to='main.Role'),
),
migrations.AlterIndexTogether(
name='roleancestorentry',
index_together=set([('descendent', 'content_type_id', 'role_field'), ('ancestor', 'content_type_id', 'role_field')]),
),
migrations.AddField( migrations.AddField(
model_name='credential', model_name='credential',

50
awx/main/models/mixins.py
View File

@@ -5,7 +5,7 @@ from django.contrib.auth.models import User # noqa
# AWX # AWX
from awx.main.models.rbac import ( from awx.main.models.rbac import (
Role, get_roles_on_resource Role, RoleAncestorEntry, get_roles_on_resource
) )
@@ -17,7 +17,7 @@ class ResourceMixin(models.Model):
abstract = True abstract = True
@classmethod @classmethod
def accessible_objects(cls, accessor, role_name): def accessible_objects(cls, accessor, role_field):
''' '''
Use instead of `MyModel.objects` when you want to only consider Use instead of `MyModel.objects` when you want to only consider
resources that a user has specific permissions for. For example: resources that a user has specific permissions for. For example:
@@ -29,44 +29,26 @@ class ResourceMixin(models.Model):
performant to resolve the resource in question then call performant to resolve the resource in question then call
`myresource.get_permissions(user)`. `myresource.get_permissions(user)`.
''' '''
return ResourceMixin._accessible_objects(cls, accessor, role_name) return ResourceMixin._accessible_objects(cls, accessor, role_field)
@staticmethod @staticmethod
def _accessible_objects(cls, accessor, role_name): def _accessible_objects(cls, accessor, role_field):
if type(cls()) == User:
cls_type = ContentType.objects.get_for_model(cls)
roles = Role.objects.filter(content_type__pk=cls_type.id)
if type(accessor) == User:
roles = roles.filter(ancestors__members = accessor)
elif type(accessor) == Role:
roles = roles.filter(ancestors = accessor)
else:
accessor_type = ContentType.objects.get_for_model(accessor)
accessor_roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id)
roles = roles.filter(ancestors__in=accessor_roles)
kwargs = {'id__in':roles.values_list('object_id', flat=True)}
return cls.objects.filter(**kwargs).distinct()
if type(accessor) == User: if type(accessor) == User:
kwargs = {} ancestor_roles = accessor.roles.all()
kwargs[role_name + '__ancestors__members'] = accessor
qs = cls.objects.filter(**kwargs)
elif type(accessor) == Role: elif type(accessor) == Role:
kwargs = {} ancestor_roles = [accessor]
kwargs[role_name + '__ancestors'] = accessor
qs = cls.objects.filter(**kwargs)
else: else:
accessor_type = ContentType.objects.get_for_model(accessor) accessor_type = ContentType.objects.get_for_model(accessor)
roles = Role.objects.filter(content_type__pk=accessor_type.id, ancestor_roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id) object_id=accessor.id)
kwargs = {} qs = cls.objects.filter(pk__in =
kwargs[role_name + '__ancestors__in'] = roles RoleAncestorEntry.objects.filter(
qs = cls.objects.filter(**kwargs) ancestor__in=ancestor_roles,
content_type_id = ContentType.objects.get_for_model(cls).id,
return qs.distinct() role_field = role_field
).values_list('object_id').distinct()
)
return qs
def get_permissions(self, accessor): def get_permissions(self, accessor):

115
awx/main/models/rbac.py
View File

@@ -80,7 +80,12 @@ class Role(CommonModelNameNotUnique):
role_field = models.TextField(null=False, default='') role_field = models.TextField(null=False, default='')
parents = models.ManyToManyField('Role', related_name='children') parents = models.ManyToManyField('Role', related_name='children')
implicit_parents = models.TextField(null=False, default='[]') implicit_parents = models.TextField(null=False, default='[]')
ancestors = models.ManyToManyField('Role', related_name='descendents') # auto-generated by `rebuild_role_ancestor_list` ancestors = models.ManyToManyField(
'Role',
through='RoleAncestorEntry',
through_fields=('descendent', 'ancestor'),
related_name='descendents'
) # auto-generated by `rebuild_role_ancestor_list`
members = models.ManyToManyField('auth.User', related_name='roles') members = models.ManyToManyField('auth.User', related_name='roles')
content_type = models.ForeignKey(ContentType, null=True, default=None) content_type = models.ForeignKey(ContentType, null=True, default=None)
object_id = models.PositiveIntegerField(null=True, default=None) object_id = models.PositiveIntegerField(null=True, default=None)
@@ -106,9 +111,6 @@ class Role(CommonModelNameNotUnique):
object_id=accessor.id) object_id=accessor.id)
return self.ancestors.filter(pk__in=roles).exists() return self.ancestors.filter(pk__in=roles).exists()
def rebuild_role_ancestor_list(self): def rebuild_role_ancestor_list(self):
''' '''
Updates our `ancestors` map to accurately reflect all of the ancestors for a role Updates our `ancestors` map to accurately reflect all of the ancestors for a role
@@ -151,25 +153,6 @@ class Role(CommonModelNameNotUnique):
Role._simultaneous_ancestry_rebuild([self.id]) Role._simultaneous_ancestry_rebuild([self.id])
@staticmethod
def _simultaneous_ancestry_rebuild2(role_ids_to_rebuild):
#all_parents = Role.parents.through.values_list('to_role_id', 'from_role_id')
'''
sql_params = {
'ancestors_table': Role.ancestors.through._meta.db_table,
'parents_table': Role.parents.through._meta.db_table,
'roles_table': Role._meta.db_table,
'ids': ','.join(str(x) for x in role_ids_to_rebuild)
}
'''
#Expand our parent list
#Expand our child list
#Construct our ancestor list for
#apply diff
@staticmethod @staticmethod
def _simultaneous_ancestry_rebuild(role_ids_to_rebuild): def _simultaneous_ancestry_rebuild(role_ids_to_rebuild):
# #
@@ -250,10 +233,6 @@ class Role(CommonModelNameNotUnique):
# updated, and so we can terminate our loop. # updated, and so we can terminate our loop.
# #
# #
# *NOTE* Keen reader may realize that there are many instances where
# fuck. cycles will never shake parents.
#
#
cursor = connection.cursor() cursor = connection.cursor()
loop_ct = 0 loop_ct = 0
@@ -274,10 +253,16 @@ class Role(CommonModelNameNotUnique):
# TODO: Test to see if not deleting any entry that has a direct # TODO: Test to see if not deleting any entry that has a direct
# correponding entry in the parents table helps reduce the processing # correponding entry in the parents table helps reduce the processing
# time significantly # time significantly
"""
cursor.execute(''' cursor.execute('''
DELETE FROM %(ancestors_table)s DELETE FROM %(ancestors_table)s
WHERE to_role_id IN (%(ids)s) WHERE ancestor_id IN (%(ids)s)
AND from_role_id != to_role_id AND descendent_id != ancestor_id
''' % sql_params)
"""
cursor.execute('''
DELETE FROM %(ancestors_table)s
WHERE ancestor_id IN (%(ids)s)
''' % sql_params) ''' % sql_params)
@@ -297,42 +282,52 @@ class Role(CommonModelNameNotUnique):
cursor.execute(''' cursor.execute('''
DELETE FROM %(ancestors_table)s DELETE FROM %(ancestors_table)s
WHERE from_role_id IN (%(ids)s) WHERE descendent_id IN (%(ids)s)
AND AND
id NOT IN ( id NOT IN (
SELECT %(ancestors_table)s.id FROM ( SELECT %(ancestors_table)s.id FROM (
SELECT parents.from_role_id from_id, ancestors.to_role_id to_id SELECT parents.from_role_id from_id, ancestors.ancestor_id to_id
FROM %(parents_table)s as parents FROM %(parents_table)s as parents
LEFT JOIN %(ancestors_table)s as ancestors LEFT JOIN %(ancestors_table)s as ancestors
ON (parents.to_role_id = ancestors.from_role_id) ON (parents.to_role_id = ancestors.descendent_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.to_role_id IS NOT NULL WHERE parents.from_role_id IN (%(ids)s) AND ancestors.ancestor_id IS NOT NULL
UNION UNION
SELECT id from_id, id to_id from %(roles_table)s WHERE id IN (%(ids)s) SELECT id from_id, id to_id from %(roles_table)s WHERE id IN (%(ids)s)
) new_ancestry_list ) new_ancestry_list
LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.from_role_id LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.descendent_id
AND new_ancestry_list.to_id = %(ancestors_table)s.to_role_id) AND new_ancestry_list.to_id = %(ancestors_table)s.ancestor_id)
WHERE %(ancestors_table)s.id IS NOT NULL WHERE %(ancestors_table)s.id IS NOT NULL
) )
''' % sql_params) ''' % sql_params)
delete_ct = cursor.rowcount delete_ct = cursor.rowcount
cursor.execute(''' cursor.execute('''
INSERT INTO %(ancestors_table)s (from_role_id, to_role_id) INSERT INTO %(ancestors_table)s (descendent_id, ancestor_id, role_field, content_type_id, object_id)
SELECT from_id, to_id FROM ( SELECT from_id, to_id, new_ancestry_list.role_field, new_ancestry_list.content_type_id, new_ancestry_list.object_id FROM (
SELECT parents.from_role_id from_id, ancestors.to_role_id to_id SELECT parents.from_role_id from_id,
ancestors.ancestor_id to_id,
roles.role_field,
COALESCE(roles.content_type_id, 0) content_type_id,
COALESCE(roles.object_id, 0) object_id
FROM %(parents_table)s as parents FROM %(parents_table)s as parents
LEFT JOIN %(ancestors_table)s as ancestors INNER JOIN %(roles_table)s as roles ON (parents.from_role_id = roles.id)
ON (parents.to_role_id = ancestors.from_role_id) LEFT OUTER JOIN %(ancestors_table)s as ancestors
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.to_role_id IS NOT NULL ON (parents.to_role_id = ancestors.descendent_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.ancestor_id IS NOT NULL
UNION UNION
SELECT id from_id, id to_id from %(roles_table)s WHERE id IN (%(ids)s) SELECT id from_id,
id to_id,
role_field,
COALESCE(content_type_id, 0) content_type_id,
COALESCE(object_id, 0) object_id
from %(roles_table)s WHERE id IN (%(ids)s)
) new_ancestry_list ) new_ancestry_list
LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.from_role_id LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.descendent_id
AND new_ancestry_list.to_id = %(ancestors_table)s.to_role_id) AND new_ancestry_list.to_id = %(ancestors_table)s.ancestor_id)
WHERE %(ancestors_table)s.id IS NULL WHERE %(ancestors_table)s.id IS NULL
''' % sql_params) ''' % sql_params)
insert_ct = cursor.rowcount insert_ct = cursor.rowcount
@@ -358,7 +353,24 @@ class Role(CommonModelNameNotUnique):
def is_ancestor_of(self, role): def is_ancestor_of(self, role):
return role.ancestors.filter(id=self.id).exists() return role.ancestors.filter(id=self.id).exists()
class RoleAncestorEntry(models.Model):
class Meta:
app_label = 'main'
verbose_name_plural = _('role_ancestors')
db_table = 'main_rbac_role_ancestors'
index_together = [
("ancestor", "content_type_id", "role_field"),
("descendent", "content_type_id", "role_field"),
]
descendent = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')
ancestor = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')
role_field = models.TextField(null=False)
#content_type_id = models.PositiveIntegerField(null=False)
#object_id = models.PositiveIntegerField(null=False)
content_type_id = models.PositiveIntegerField(null=False)
object_id = models.PositiveIntegerField(null=False)
def get_roles_on_resource(resource, accessor): def get_roles_on_resource(resource, accessor):
@@ -371,15 +383,18 @@ def get_roles_on_resource(resource, accessor):
if type(accessor) == User: if type(accessor) == User:
roles = accessor.roles.all() roles = accessor.roles.all()
elif type(accessor) == Role: elif type(accessor) == Role:
roles = accessor roles = [accessor]
else: else:
accessor_type = ContentType.objects.get_for_model(accessor) accessor_type = ContentType.objects.get_for_model(accessor)
roles = Role.objects.filter(content_type__pk=accessor_type.id, roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id) object_id=accessor.id)
return { role.role_field: True for role in return {
Role.objects.filter( role_field: True for role_field in
content_type = ContentType.objects.get_for_model(resource), RoleAncestorEntry.objects.filter(
object_id = resource.id, ancestor__in=roles,
ancestors = roles)} content_type_id=ContentType.objects.get_for_model(resource).id,
object_id=resource.id
).values_list('role_field', flat=True)
}