From a6f735b4b9e4b8c837d6cf324c5b7b7ab7b88b85 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Wed, 27 Apr 2016 15:27:52 -0400 Subject: [PATCH 1/7] Fixed call to role rebuilder during migrations --- awx/main/fields.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/awx/main/fields.py b/awx/main/fields.py index d08bbdfb14..b77452ca5b 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -20,9 +20,10 @@ from django.db.models.fields.related import ( from django.utils.encoding import smart_text # AWX -from awx.main.models.rbac import batch_role_ancestor_rebuilding +from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role from awx.main.utils import get_current_apps + __all__ = ['AutoOneToOneField', 'ImplicitRoleField'] @@ -199,7 +200,7 @@ class ImplicitRoleField(models.ForeignKey): updates[role.role_field] = role.id role_ids.append(role.id) type(instance).objects.filter(pk=instance.pk).update(**updates) - Role_.rebuild_role_ancestor_list(role_ids, []) + Role.rebuild_role_ancestor_list(role_ids, []) # Update parentage if necessary for implicit_role_field in getattr(instance.__class__, '__implicit_role_fields'): @@ -255,4 +256,4 @@ class ImplicitRoleField(models.ForeignKey): Role_ = get_current_apps().get_model('main', 'Role') child_ids = [x for x in Role_.parents.through.objects.filter(to_role_id__in=role_ids).distinct().values_list('from_role_id', flat=True)] Role_.objects.filter(id__in=role_ids).delete() - Role_.rebuild_role_ancestor_list([], child_ids) + Role.rebuild_role_ancestor_list([], child_ids) From b670681f6c13a27184504d99452f21c2896a7e62 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Wed, 27 Apr 2016 15:28:17 -0400 Subject: [PATCH 2/7] Handle JT "create" permissions during migration --- awx/main/migrations/_rbac.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/migrations/_rbac.py b/awx/main/migrations/_rbac.py index 8710f8c772..3570e0184a 100644 --- a/awx/main/migrations/_rbac.py +++ b/awx/main/migrations/_rbac.py @@ -201,7 +201,7 @@ def migrate_inventory(apps, schema_editor): return inventory.auditor_role elif perm.permission_type == 'write': return inventory.update_role - elif perm.permission_type == 'check' or perm.permission_type == 'run': + elif perm.permission_type == 'check' or perm.permission_type == 'run' or perm.permission_type == 'create': # These permission types are handled differntly in RBAC now, nothing to migrate. return False else: From af4daec31414cfeffbc7352f0823c698784d2467 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Wed, 27 Apr 2016 16:59:33 -0400 Subject: [PATCH 3/7] More RBAC migration fixes --- .../migrations/0009_v300_rbac_migrations.py | 4 ++- .../0017_v300_prompting_migrations.py | 2 ++ awx/main/migrations/_migration_utils.py | 11 ++++++++ awx/main/migrations/_rbac.py | 28 +++++++++++++++---- 4 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 awx/main/migrations/_migration_utils.py diff --git a/awx/main/migrations/0009_v300_rbac_migrations.py b/awx/main/migrations/0009_v300_rbac_migrations.py index 9c7f7d8dd7..dd26690b22 100644 --- a/awx/main/migrations/0009_v300_rbac_migrations.py +++ b/awx/main/migrations/0009_v300_rbac_migrations.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from awx.main.migrations import _rbac as rbac +from awx.main.migrations import _migration_utils as migration_utils from django.db import migrations @@ -12,11 +13,12 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(rbac.init_rbac_migration), + migrations.RunPython(migration_utils.set_current_apps_for_migrations), migrations.RunPython(rbac.migrate_users), migrations.RunPython(rbac.migrate_organization), migrations.RunPython(rbac.migrate_team), migrations.RunPython(rbac.migrate_inventory), migrations.RunPython(rbac.migrate_projects), migrations.RunPython(rbac.migrate_credential), + migrations.RunPython(rbac.rebuild_role_hierarchy), ] diff --git a/awx/main/migrations/0017_v300_prompting_migrations.py b/awx/main/migrations/0017_v300_prompting_migrations.py index f08d760a8c..c5a1df0eb9 100644 --- a/awx/main/migrations/0017_v300_prompting_migrations.py +++ b/awx/main/migrations/0017_v300_prompting_migrations.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from awx.main.migrations import _ask_for_variables as ask_for_variables +from awx.main.migrations import _migration_utils as migration_utils from django.db import migrations @@ -12,5 +13,6 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RunPython(migration_utils.set_current_apps_for_migrations), migrations.RunPython(ask_for_variables.migrate_credential), ] diff --git a/awx/main/migrations/_migration_utils.py b/awx/main/migrations/_migration_utils.py new file mode 100644 index 0000000000..232310ca50 --- /dev/null +++ b/awx/main/migrations/_migration_utils.py @@ -0,0 +1,11 @@ +from awx.main.utils import set_current_apps + + +def set_current_apps_for_migrations(apps, schema_editor): + ''' + This is necessary for migrations which do explicit saves on any model that + has an ImplicitRoleFIeld (which generally means anything that has + some RBAC bindings associated with it). This sets the current 'apps' that + the ImplicitRoleFIeld should be using when creating new roles. + ''' + set_current_apps(apps) diff --git a/awx/main/migrations/_rbac.py b/awx/main/migrations/_rbac.py index 3570e0184a..0a816610ed 100644 --- a/awx/main/migrations/_rbac.py +++ b/awx/main/migrations/_rbac.py @@ -1,11 +1,13 @@ import logging +from time import time from django.utils.encoding import smart_text from django.db.models import Q -from django.utils.timezone import now +#from django.utils.timezone import now from collections import defaultdict -from awx.main.utils import getattrd, set_current_apps +from awx.main.utils import getattrd +from awx.main.models.rbac import Role import _old_access as old_access logger = logging.getLogger(__name__) @@ -26,9 +28,6 @@ def log_migration(wrapped): return wrapped(*args, **kwargs) return wrapper -@log_migration -def init_rbac_migration(apps, schema_editor): - set_current_apps(apps) @log_migration def migrate_users(apps, schema_editor): @@ -395,3 +394,22 @@ def migrate_job_templates(apps, schema_editor): if old_access.check_user_access(user, jt.__class__, 'start', jt, False): jt.execute_role.members.add(user) logger.info(smart_text(u'adding User({}) access to JobTemplate({})'.format(user.username, jt.name))) + +@log_migration +def rebuild_role_hierarchy(apps, schema_editor): + logger.info('Computing role roots..') + start = time() + roots = Role.objects \ + .all() \ + .exclude(pk__in=Role.parents.through.objects.all() + .values_list('from_role_id', flat=True).distinct()) \ + .values_list('id', flat=True) + stop = time() + logger.info('Found %d roots in %f seconds, rebuilding ancestry map' % (len(roots), stop - start)) + start = time() + Role.rebuild_role_ancestor_list(roots, []) + stop = time() + logger.info('Rebuild completed in %f seconds' % (stop - start)) + logger.info('Done.') + + From c9501ad556ce35d90cfe253436d33fe74490fd81 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Thu, 28 Apr 2016 09:39:10 -0400 Subject: [PATCH 4/7] commented out code cleanup --- awx/main/migrations/_rbac.py | 1 - 1 file changed, 1 deletion(-) diff --git a/awx/main/migrations/_rbac.py b/awx/main/migrations/_rbac.py index 0a816610ed..4b76848748 100644 --- a/awx/main/migrations/_rbac.py +++ b/awx/main/migrations/_rbac.py @@ -3,7 +3,6 @@ from time import time from django.utils.encoding import smart_text from django.db.models import Q -#from django.utils.timezone import now from collections import defaultdict from awx.main.utils import getattrd From 27a7cf0d8892bc2ed18343c528fd01f6e474d714 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Thu, 28 Apr 2016 10:37:15 -0400 Subject: [PATCH 5/7] Added an explicit save everything step in the rbac migration This step ensures all of our roles get setup before we start the migration. It also speeds things up a little as we can wrap everything with a `with batch_role_ancestor_rebuilding()`. We were pretty much doing this already, but we had an issue where we didn't catch all job templates all the time, so this just makes it very explicit and ensures everything is setup, and does so a little faster. --- .../migrations/0009_v300_rbac_migrations.py | 1 + awx/main/migrations/_rbac.py | 38 +++++++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/awx/main/migrations/0009_v300_rbac_migrations.py b/awx/main/migrations/0009_v300_rbac_migrations.py index dd26690b22..575eeb02a7 100644 --- a/awx/main/migrations/0009_v300_rbac_migrations.py +++ b/awx/main/migrations/0009_v300_rbac_migrations.py @@ -15,6 +15,7 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(migration_utils.set_current_apps_for_migrations), migrations.RunPython(rbac.migrate_users), + migrations.RunPython(rbac.create_roles), migrations.RunPython(rbac.migrate_organization), migrations.RunPython(rbac.migrate_team), migrations.RunPython(rbac.migrate_inventory), diff --git a/awx/main/migrations/_rbac.py b/awx/main/migrations/_rbac.py index 4b76848748..465498d365 100644 --- a/awx/main/migrations/_rbac.py +++ b/awx/main/migrations/_rbac.py @@ -6,7 +6,7 @@ from django.db.models import Q from collections import defaultdict from awx.main.utils import getattrd -from awx.main.models.rbac import Role +from awx.main.models.rbac import Role, batch_role_ancestor_rebuilding import _old_access as old_access logger = logging.getLogger(__name__) @@ -27,6 +27,36 @@ def log_migration(wrapped): return wrapped(*args, **kwargs) return wrapper +@log_migration +def create_roles(apps, schema_editor): + ''' + Implicit role creation happens in our post_save hook for all of our + resources. Here we iterate through all of our resource types and call + .save() to ensure all that happens for every object in the system before we + get busy with the actual migration work. + + This gets run after migrate_users, which does role creation for users a + little differently. + ''' + + models = [ + apps.get_model('main', m) for m in [ + 'Organization', + 'Team', + 'Inventory', + 'Group', + 'Project', + 'Credential', + 'CustomInventoryScript', + 'JobTemplate', + ] + ] + + with batch_role_ancestor_rebuilding(): + for model in models: + for obj in model.objects.iterator(): + obj.save() + @log_migration def migrate_users(apps, schema_editor): @@ -65,7 +95,6 @@ def migrate_users(apps, schema_editor): def migrate_organization(apps, schema_editor): Organization = apps.get_model('main', "Organization") for org in Organization.objects.iterator(): - org.save() # force creates missing roles for admin in org.deprecated_admins.all(): org.admin_role.members.add(admin) logger.info(smart_text(u"added admin: {}, {}".format(org.name, admin.username))) @@ -77,7 +106,6 @@ def migrate_organization(apps, schema_editor): def migrate_team(apps, schema_editor): Team = apps.get_model('main', 'Team') for t in Team.objects.iterator(): - t.save() for user in t.deprecated_users.all(): t.member_role.members.add(user) logger.info(smart_text(u"team: {}, added user: {}".format(t.name, user.username))) @@ -151,7 +179,6 @@ def migrate_credential(apps, schema_editor): user_content_type = ContentType.objects.get_for_model(User) for cred in Credential.objects.iterator(): - cred.save() results = (JobTemplate.objects.filter(Q(credential=cred) | Q(cloud_credential=cred)).all() or InventorySource.objects.filter(credential=cred).all()) if results: @@ -206,7 +233,6 @@ def migrate_inventory(apps, schema_editor): return None for inventory in Inventory.objects.iterator(): - inventory.save() for perm in Permission.objects.filter(inventory=inventory): role = None execrole = None @@ -254,7 +280,6 @@ def migrate_projects(apps, schema_editor): # Migrate projects to single organizations, duplicating as necessary for project in Project.objects.iterator(): - project.save() original_project_name = project.name project_orgs = project.deprecated_organizations.distinct().all() @@ -367,7 +392,6 @@ def migrate_job_templates(apps, schema_editor): Permission = apps.get_model('main', 'Permission') for jt in JobTemplate.objects.iterator(): - jt.save() permission = Permission.objects.filter( inventory=jt.inventory, project=jt.project, From ac1503e55c77de1b5b631a01df8f2bf9cce4ad52 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Thu, 28 Apr 2016 11:50:36 -0400 Subject: [PATCH 6/7] Fixed org migrations.. users are 'members' not 'auditors' --- awx/main/migrations/_rbac.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/main/migrations/_rbac.py b/awx/main/migrations/_rbac.py index 465498d365..16513377df 100644 --- a/awx/main/migrations/_rbac.py +++ b/awx/main/migrations/_rbac.py @@ -99,8 +99,8 @@ def migrate_organization(apps, schema_editor): org.admin_role.members.add(admin) logger.info(smart_text(u"added admin: {}, {}".format(org.name, admin.username))) for user in org.deprecated_users.all(): - org.auditor_role.members.add(user) - logger.info(smart_text(u"added auditor: {}, {}".format(org.name, user.username))) + org.member_role.members.add(user) + logger.info(smart_text(u"added member: {}, {}".format(org.name, user.username))) @log_migration def migrate_team(apps, schema_editor): From 6427526686dbe82728f69a729fe88ba972f12329 Mon Sep 17 00:00:00 2001 From: Akita Noek Date: Fri, 29 Apr 2016 15:34:54 -0400 Subject: [PATCH 7/7] Updated JT migration tests to be more correct This test was passing before because we were erroneously making all users organization auditors, which gave users read access to all JT's under the org. --- awx/main/tests/functional/test_rbac_job_templates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awx/main/tests/functional/test_rbac_job_templates.py b/awx/main/tests/functional/test_rbac_job_templates.py index 93538f67f0..e439533f07 100644 --- a/awx/main/tests/functional/test_rbac_job_templates.py +++ b/awx/main/tests/functional/test_rbac_job_templates.py @@ -87,7 +87,7 @@ def test_job_template_team_migration_check(deploy_jobtemplate, check_jobtemplate rbac.migrate_projects(apps, None) rbac.migrate_inventory(apps, None) - assert joe in check_jobtemplate.read_role + assert joe not in check_jobtemplate.read_role assert admin in check_jobtemplate.execute_role assert joe not in check_jobtemplate.execute_role @@ -120,12 +120,13 @@ def test_job_template_team_deploy_migration(deploy_jobtemplate, check_jobtemplat rbac.migrate_projects(apps, None) rbac.migrate_inventory(apps, None) - assert joe in deploy_jobtemplate.read_role + assert joe not in deploy_jobtemplate.read_role assert admin in deploy_jobtemplate.execute_role assert joe not in deploy_jobtemplate.execute_role rbac.migrate_job_templates(apps, None) + assert joe in deploy_jobtemplate.read_role assert admin in deploy_jobtemplate.execute_role assert joe in deploy_jobtemplate.execute_role