External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-14 19:30:39 -03:30
Code Issues Packages Projects Releases Wiki Activity

Switch to custom ancestry table for some optimized queries

Browse Source 
Now we can stuff some more data in this table so we can take advantage
of some multi-column indexing and avoid another to join for our
accessible objects and permissions queries.
This commit is contained in:
Akita Noek 2016-04-16 18:27:57 -04:00
parent 25303cf4ec
commit 5d0c6cc044
3 changed files with 109 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
awx/main/migrations/0008_v300_rbac_changes.py
View File

@ -48,6 +48,19 @@ class Migration(migrations.Migration):
field=models.ManyToManyField(related_name='deprecated_teams', to='main.Project', blank=True),
),
migrations.CreateModel(
name='RoleAncestorEntry',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('role_field', models.TextField()),
('content_type_id', models.PositiveIntegerField(null=False)),
('object_id', models.PositiveIntegerField(null=False)),
],
options={
'db_table': 'main_rbac_role_ancestors',
'verbose_name_plural': 'role_ancestors',
},
),
migrations.CreateModel(
name='Role',
fields=[
@ -58,7 +71,7 @@ class Migration(migrations.Migration):
('name', models.CharField(max_length=512)),
('singleton_name', models.TextField(default=None, unique=True, null=True, db_index=True)),
('object_id', models.PositiveIntegerField(default=None, null=True)),
('ancestors', models.ManyToManyField(related_name='descendents', to='main.Role')),
('ancestors', models.ManyToManyField(related_name='descendents', through='main.RoleAncestorEntry', to='main.Role')),
('content_type', models.ForeignKey(default=None, to='contenttypes.ContentType', null=True)),
('created_by', models.ForeignKey(related_name="{u'class': 'role', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
('members', models.ManyToManyField(related_name='roles', to=settings.AUTH_USER_MODEL)),
@ -72,6 +85,20 @@ class Migration(migrations.Migration):
'verbose_name_plural': 'roles',
},
),
migrations.AddField(
model_name='roleancestorentry',
name='ancestor',
field=models.ForeignKey(related_name='+', to='main.Role'),
),
migrations.AddField(
model_name='roleancestorentry',
name='descendent',
field=models.ForeignKey(related_name='+', to='main.Role'),
),
migrations.AlterIndexTogether(
name='roleancestorentry',
index_together=set([('descendent', 'content_type_id', 'role_field'), ('ancestor', 'content_type_id', 'role_field')]),
),
migrations.AddField(
model_name='credential',

50
awx/main/models/mixins.py
View File

@ -5,7 +5,7 @@ from django.contrib.auth.models import User # noqa
# AWX
from awx.main.models.rbac import (
Role, get_roles_on_resource
Role, RoleAncestorEntry, get_roles_on_resource
)
@ -17,7 +17,7 @@ class ResourceMixin(models.Model):
abstract = True
@classmethod
def accessible_objects(cls, accessor, role_name):
def accessible_objects(cls, accessor, role_field):
'''
Use instead of `MyModel.objects` when you want to only consider
resources that a user has specific permissions for. For example:
@ -29,44 +29,26 @@ class ResourceMixin(models.Model):
performant to resolve the resource in question then call
`myresource.get_permissions(user)`.
'''
return ResourceMixin._accessible_objects(cls, accessor, role_name)
return ResourceMixin._accessible_objects(cls, accessor, role_field)
@staticmethod
def _accessible_objects(cls, accessor, role_name):
if type(cls()) == User:
cls_type = ContentType.objects.get_for_model(cls)
roles = Role.objects.filter(content_type__pk=cls_type.id)
if type(accessor) == User:
roles = roles.filter(ancestors__members = accessor)
elif type(accessor) == Role:
roles = roles.filter(ancestors = accessor)
else:
accessor_type = ContentType.objects.get_for_model(accessor)
accessor_roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id)
roles = roles.filter(ancestors__in=accessor_roles)
kwargs = {'id__in':roles.values_list('object_id', flat=True)}
return cls.objects.filter(**kwargs).distinct()
def _accessible_objects(cls, accessor, role_field):
if type(accessor) == User:
kwargs = {}
kwargs[role_name + '__ancestors__members'] = accessor
qs = cls.objects.filter(**kwargs)
ancestor_roles = accessor.roles.all()
elif type(accessor) == Role:
kwargs = {}
kwargs[role_name + '__ancestors'] = accessor
qs = cls.objects.filter(**kwargs)
ancestor_roles = [accessor]
else:
accessor_type = ContentType.objects.get_for_model(accessor)
roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id)
kwargs = {}
kwargs[role_name + '__ancestors__in'] = roles
qs = cls.objects.filter(**kwargs)
return qs.distinct()
ancestor_roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id)
qs = cls.objects.filter(pk__in =
RoleAncestorEntry.objects.filter(
ancestor__in=ancestor_roles,
content_type_id = ContentType.objects.get_for_model(cls).id,
role_field = role_field
).values_list('object_id').distinct()
)
return qs
def get_permissions(self, accessor):

115
awx/main/models/rbac.py
View File

@ -80,7 +80,12 @@ class Role(CommonModelNameNotUnique):
role_field = models.TextField(null=False, default='')
parents = models.ManyToManyField('Role', related_name='children')
implicit_parents = models.TextField(null=False, default='[]')
ancestors = models.ManyToManyField('Role', related_name='descendents') # auto-generated by `rebuild_role_ancestor_list`
ancestors = models.ManyToManyField(
'Role',
through='RoleAncestorEntry',
through_fields=('descendent', 'ancestor'),
related_name='descendents'
) # auto-generated by `rebuild_role_ancestor_list`
members = models.ManyToManyField('auth.User', related_name='roles')
content_type = models.ForeignKey(ContentType, null=True, default=None)
object_id = models.PositiveIntegerField(null=True, default=None)
@ -106,9 +111,6 @@ class Role(CommonModelNameNotUnique):
object_id=accessor.id)
return self.ancestors.filter(pk__in=roles).exists()
def rebuild_role_ancestor_list(self):
'''
Updates our `ancestors` map to accurately reflect all of the ancestors for a role
@ -151,25 +153,6 @@ class Role(CommonModelNameNotUnique):
Role._simultaneous_ancestry_rebuild([self.id])
@staticmethod
def _simultaneous_ancestry_rebuild2(role_ids_to_rebuild):
#all_parents = Role.parents.through.values_list('to_role_id', 'from_role_id')
'''
sql_params = {
'ancestors_table': Role.ancestors.through._meta.db_table,
'parents_table': Role.parents.through._meta.db_table,
'roles_table': Role._meta.db_table,
'ids': ','.join(str(x) for x in role_ids_to_rebuild)
}
'''
#Expand our parent list
#Expand our child list
#Construct our ancestor list for
#apply diff
@staticmethod
def _simultaneous_ancestry_rebuild(role_ids_to_rebuild):
#
@ -250,10 +233,6 @@ class Role(CommonModelNameNotUnique):
# updated, and so we can terminate our loop.
#
#
# *NOTE* Keen reader may realize that there are many instances where
# fuck. cycles will never shake parents.
#
#
cursor = connection.cursor()
loop_ct = 0
@ -274,10 +253,16 @@ class Role(CommonModelNameNotUnique):
# TODO: Test to see if not deleting any entry that has a direct
# correponding entry in the parents table helps reduce the processing
# time significantly
"""
cursor.execute('''
DELETE FROM %(ancestors_table)s
WHERE to_role_id IN (%(ids)s)
AND from_role_id != to_role_id
WHERE ancestor_id IN (%(ids)s)
AND descendent_id != ancestor_id
''' % sql_params)
"""
cursor.execute('''
DELETE FROM %(ancestors_table)s
WHERE ancestor_id IN (%(ids)s)
''' % sql_params)
@ -297,42 +282,52 @@ class Role(CommonModelNameNotUnique):
cursor.execute('''
DELETE FROM %(ancestors_table)s
WHERE from_role_id IN (%(ids)s)
WHERE descendent_id IN (%(ids)s)
AND
id NOT IN (
SELECT %(ancestors_table)s.id FROM (
SELECT parents.from_role_id from_id, ancestors.to_role_id to_id
SELECT parents.from_role_id from_id, ancestors.ancestor_id to_id
FROM %(parents_table)s as parents
LEFT JOIN %(ancestors_table)s as ancestors
ON (parents.to_role_id = ancestors.from_role_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.to_role_id IS NOT NULL
ON (parents.to_role_id = ancestors.descendent_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.ancestor_id IS NOT NULL
UNION
SELECT id from_id, id to_id from %(roles_table)s WHERE id IN (%(ids)s)
) new_ancestry_list
LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.from_role_id
AND new_ancestry_list.to_id = %(ancestors_table)s.to_role_id)
LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.descendent_id
AND new_ancestry_list.to_id = %(ancestors_table)s.ancestor_id)
WHERE %(ancestors_table)s.id IS NOT NULL
)
''' % sql_params)
delete_ct = cursor.rowcount
cursor.execute('''
INSERT INTO %(ancestors_table)s (from_role_id, to_role_id)
SELECT from_id, to_id FROM (
SELECT parents.from_role_id from_id, ancestors.to_role_id to_id
INSERT INTO %(ancestors_table)s (descendent_id, ancestor_id, role_field, content_type_id, object_id)
SELECT from_id, to_id, new_ancestry_list.role_field, new_ancestry_list.content_type_id, new_ancestry_list.object_id FROM (
SELECT parents.from_role_id from_id,
ancestors.ancestor_id to_id,
roles.role_field,
COALESCE(roles.content_type_id, 0) content_type_id,
COALESCE(roles.object_id, 0) object_id
FROM %(parents_table)s as parents
LEFT JOIN %(ancestors_table)s as ancestors
ON (parents.to_role_id = ancestors.from_role_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.to_role_id IS NOT NULL
INNER JOIN %(roles_table)s as roles ON (parents.from_role_id = roles.id)
LEFT OUTER JOIN %(ancestors_table)s as ancestors
ON (parents.to_role_id = ancestors.descendent_id)
WHERE parents.from_role_id IN (%(ids)s) AND ancestors.ancestor_id IS NOT NULL
UNION
SELECT id from_id, id to_id from %(roles_table)s WHERE id IN (%(ids)s)
SELECT id from_id,
id to_id,
role_field,
COALESCE(content_type_id, 0) content_type_id,
COALESCE(object_id, 0) object_id
from %(roles_table)s WHERE id IN (%(ids)s)
) new_ancestry_list
LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.from_role_id
AND new_ancestry_list.to_id = %(ancestors_table)s.to_role_id)
LEFT JOIN %(ancestors_table)s ON (new_ancestry_list.from_id = %(ancestors_table)s.descendent_id
AND new_ancestry_list.to_id = %(ancestors_table)s.ancestor_id)
WHERE %(ancestors_table)s.id IS NULL
''' % sql_params)
insert_ct = cursor.rowcount
@ -358,7 +353,24 @@ class Role(CommonModelNameNotUnique):
def is_ancestor_of(self, role):
return role.ancestors.filter(id=self.id).exists()
class RoleAncestorEntry(models.Model):
class Meta:
app_label = 'main'
verbose_name_plural = _('role_ancestors')
db_table = 'main_rbac_role_ancestors'
index_together = [
("ancestor", "content_type_id", "role_field"),
("descendent", "content_type_id", "role_field"),
]
descendent = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')
ancestor = models.ForeignKey(Role, null=False, on_delete=models.CASCADE, related_name='+')
role_field = models.TextField(null=False)
#content_type_id = models.PositiveIntegerField(null=False)
#object_id = models.PositiveIntegerField(null=False)
content_type_id = models.PositiveIntegerField(null=False)
object_id = models.PositiveIntegerField(null=False)
def get_roles_on_resource(resource, accessor):
@ -371,15 +383,18 @@ def get_roles_on_resource(resource, accessor):
if type(accessor) == User:
roles = accessor.roles.all()
elif type(accessor) == Role:
roles = accessor
roles = [accessor]
else:
accessor_type = ContentType.objects.get_for_model(accessor)
roles = Role.objects.filter(content_type__pk=accessor_type.id,
object_id=accessor.id)
return { role.role_field: True for role in
Role.objects.filter(
content_type = ContentType.objects.get_for_model(resource),
object_id = resource.id,
ancestors = roles)}
return {
role_field: True for role_field in
RoleAncestorEntry.objects.filter(
ancestor__in=roles,
content_type_id=ContentType.objects.get_for_model(resource).id,
object_id=resource.id
).values_list('role_field', flat=True)
}