Refactor of accessible_objects for polymorphic model

query simplification, with break-out into an intermediary
state that can be used in access methods

awx/main/access.py

@@ -1825,25 +1825,22 @@ class JobEventAccess(BaseAccess):
 class UnifiedJobTemplateAccess(BaseAccess):
     '''
     I can see a unified job template whenever I can see the same project,
-    inventory source or job template.  Unified job templates do not include
+    inventory source, WFJT, or job template.  Unified job templates do not include
-    projects without SCM configured or inventory sources without a cloud
+    inventory sources without a cloud source.
-    source.
     '''
 
     model = UnifiedJobTemplate
 
     def get_queryset(self):
-        qs = self.model.objects.all()
+        if self.user.is_superuser or self.user.is_system_auditor:
-        project_qs = self.user.get_queryset(Project).filter(scm_type__in=[s[0] for s in Project.SCM_TYPE_CHOICES])
+            qs = self.model.objects.all()
-        inventory_source_qs = self.user.get_queryset(InventorySource).filter(source__in=CLOUD_INVENTORY_SOURCES)
+        else:
-        job_template_qs = self.user.get_queryset(JobTemplate)
+            qs = self.model.objects.filter(
-        system_job_template_qs = self.user.get_queryset(SystemJobTemplate)
+                Q(pk__in=self.model.accessible_pk_qs(self.user, 'read_role')) |
-        workflow_job_template_qs = self.user.get_queryset(WorkflowJobTemplate)
+                Q(inventorysource__inventory__id__in=Inventory._accessible_pk_qs(
-        qs = qs.filter(Q(Project___in=project_qs) |
+                    Inventory, self.user, 'read_role')))
-                       Q(InventorySource___in=inventory_source_qs) |
+        qs = qs.exclude(inventorysource__source="")
-                       Q(JobTemplate___in=job_template_qs) |
+
-                       Q(systemjobtemplate__in=system_job_template_qs) |
-                       Q(workflowjobtemplate__in=workflow_job_template_qs))
         qs = qs.select_related(
             'created_by',
             'modified_by',
@@ -1882,25 +1879,25 @@ class UnifiedJobAccess(BaseAccess):
     model = UnifiedJob
 
     def get_queryset(self):
-        qs = self.model.objects.all()
+        if self.user.is_superuser or self.user.is_system_auditor:
-        project_update_qs = self.user.get_queryset(ProjectUpdate)
+            qs = self.model.objects.all()
-        inventory_update_qs = self.user.get_queryset(InventoryUpdate).filter(source__in=CLOUD_INVENTORY_SOURCES)
+        else:
-        job_qs = self.user.get_queryset(Job)
+            inv_pk_qs = Inventory._accessible_pk_qs(Inventory, self.user, 'read_role')
-        ad_hoc_command_qs = self.user.get_queryset(AdHocCommand)
+            inv_update_qs = InventoryUpdate.objects.filter(inventory_source__inventory__id__in=inv_pk_qs)
-        system_job_qs = self.user.get_queryset(SystemJob)
+            ad_hoc_command_qs = AdHocCommand.objects.filter(inventory__id__in=inv_pk_qs)
-        workflow_job_qs = self.user.get_queryset(WorkflowJob)
+            org_auditor_qs = Organization.objects.filter(
-        qs = qs.filter(Q(ProjectUpdate___in=project_update_qs) |
+                Q(admin_role__members=self.user) | Q(auditor_role__members=self.user))
-                       Q(InventoryUpdate___in=inventory_update_qs) |
+            qs = self.model.objects.filter(
-                       Q(Job___in=job_qs) |
+                Q(unified_job_template__id__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role')) |
-                       Q(AdHocCommand___in=ad_hoc_command_qs) |
+                Q(inventoryupdate__in=inv_update_qs) |
-                       Q(SystemJob___in=system_job_qs) |
+                Q(adhoccommand__in=ad_hoc_command_qs) |
-                       Q(WorkflowJob___in=workflow_job_qs))
+                Q(job__inventory__organization__in=org_auditor_qs) |
-        qs = qs.select_related(
+                Q(job__project__organization__in=org_auditor_qs)
+            )
+        qs = qs.prefetch_related(
             'created_by',
             'modified_by',
             'unified_job_node__workflow_job',
-        )
-        qs = qs.prefetch_related(
             'unified_job_template',
         )
 
@@ -1922,6 +1919,9 @@ class UnifiedJobAccess(BaseAccess):
         #    'job_template__credential',
         #    'job_template__cloud_credential',
         #)
+        # Maybe we can do these, like:
+        # 'projectupdate__project',
+        # 'inventoryupdate__inventory'
         return qs.all()
 
 

awx/main/models/mixins.py

@@ -3,6 +3,7 @@ import json
 
 # Django
 from django.db import models
+from django.db.models import Q
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.auth.models import User # noqa
 
@@ -37,8 +38,9 @@ class ResourceMixin(models.Model):
         '''
         return ResourceMixin._accessible_objects(cls, accessor, role_field)
 
+
     @staticmethod
-    def _accessible_objects(cls, accessor, role_field):
+    def _accessible_pk_qs(cls, accessor, role_field, content_types=None):
         if type(accessor) == User:
             ancestor_roles = accessor.roles.all()
         elif type(accessor) == Role:
@@ -47,14 +49,24 @@ class ResourceMixin(models.Model):
             accessor_type = ContentType.objects.get_for_model(accessor)
             ancestor_roles = Role.objects.filter(content_type__pk=accessor_type.id,
                                                  object_id=accessor.id)
-        qs = cls.objects.filter(pk__in =
+
-                                RoleAncestorEntry.objects.filter(
+        if content_types is not None:
-                                    ancestor__in=ancestor_roles,
+            return RoleAncestorEntry.objects.filter(
-                                    content_type_id = ContentType.objects.get_for_model(cls).id,
+                ancestor__in = ancestor_roles,
-                                    role_field = role_field
+                content_type_id__in = content_types,
-                                ).values_list('object_id').distinct()
+                role_field = role_field
-                                )
+            ).values_list('object_id').distinct()
-        return qs
+
+        return RoleAncestorEntry.objects.filter(
+            ancestor__in = ancestor_roles,
+            content_type_id = ContentType.objects.get_for_model(cls).id,
+            role_field = role_field
+        ).values_list('object_id').distinct()
+
+
+    @staticmethod
+    def _accessible_objects(cls, accessor, role_field):
+        return cls.objects.filter(pk__in = ResourceMixin._accessible_pk_qs(cls, accessor, role_field))
 
 
     def get_permissions(self, accessor):

awx/main/models/unified_jobs.py

@@ -20,6 +20,7 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils.timezone import now
 from django.utils.encoding import smart_text
 from django.apps import apps
+from django.contrib.contenttypes.models import ContentType
 
 # Django-Polymorphic
 from polymorphic import PolymorphicModel
@@ -30,6 +31,7 @@ from djcelery.models import TaskMeta
 # AWX
 from awx.main.models.base import * # noqa
 from awx.main.models.schedules import Schedule
+from awx.main.models.mixins import ResourceMixin
 from awx.main.utils import (
     decrypt_field, _inventory_updates,
     copy_model_by_class, copy_m2m_relationships
@@ -166,6 +168,22 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio
         else:
             return super(UnifiedJobTemplate, self).unique_error_message(model_class, unique_check)
 
+    @classmethod
+    def accessible_pk_qs(cls, accessor, role_field):
+        '''
+        A re-implementation of accessible pk queryset for the "normal" unified JTs.
+        Does not return inventory sources or system JTs, these should
+        be handled inside of get_queryset where it is utilized.
+        '''
+        # Algorithmic option, if we don't want hardcoded class names
+        # ujt_name_list = [c.__name__.lower() for c in cls.__subclasses__()]
+        # ujt_name_list.remove('inventorysource')
+        subclass_content_types = ContentType.objects.filter(
+            model__in=['project', 'jobtemplate', 'systemjobtemplate', 'workflowjobtemplate']
+        ).values_list('id', flat=True)
+
+        return ResourceMixin._accessible_pk_qs(cls, accessor, role_field, content_types=subclass_content_types)
+
     def _perform_unique_checks(self, unique_checks):
         # Handle the list of unique fields returned above. Replace with an
         # appropriate error message for the remaining field(s) in the unique

tools/data_generators/presets.tsv

@@ -1,15 +1,15 @@
-resource	medium
+resource	medium	jan2017
-organizations	500
+organizations	500	1
-users	5000
+users	5000	3
-teams	500
+teams	500	2
-projects	1000
+projects	1000	30
-job-templates	2000
+job-templates	2000	127
-credentials	2000
+credentials	2000	50
-inventories	2000
+inventories	2000	6
-inventory-groups	500
+inventory-groups	500	15
-inventory-hosts	2500
+inventory-hosts	2500	15
-wfjts	100
+wfjts	100	0
-nodes	1000
+nodes	1000	0
-labels	1000
+labels	1000	0
-jobs	1000
+jobs	1000	157208
-job-events	1000
+job-events	1000	3370942

tools/data_generators/rbac_dummy_data_generator.py

@@ -7,6 +7,7 @@ import sys
 # Python
 from collections import defaultdict
 from optparse import make_option, OptionParser
+import logging
 
 
 # Django
@@ -84,6 +85,7 @@ options = vars(options)
 
 
 if options['preset']:
+    print ' Using preset data numbers set ' + str(options['preset'])
     # Read the numbers of resources from presets file, if provided
     presets_filename = os.path.abspath(os.path.join(
         os.path.dirname(os.path.abspath(__file__)), 'presets.tsv'))
@@ -603,22 +605,28 @@ try:
                     wfjt.labels.add(next(label_gen))
                 wfjt_idx += 1
 
+            # Disable logging here, because it will mess up output format
+            logger = logging.getLogger('awx.main')
+            logger.propagate = False
+
             print('# Creating %d jobs' % n_jobs)
             group_idx = 0
             job_template_idx = 0
+            job_i = 0
             for n in spread(n_jobs, n_job_templates):
                 job_template = job_templates[job_template_idx]
                 for i in range(n):
                     sys.stdout.write('\r   Assigning %d to %s: %d     ' % (n, job_template.name, i+ 1))
                     sys.stdout.flush()
-                    job_stat = 'successful'
                     if len(jobs) % 4 == 0:
                         job_stat = 'failed'
                     elif len(jobs) % 11 == 0:
                         job_stat = 'canceled'
+                    else:
+                        job_stat = 'successful'
                     job, _ = Job.objects.get_or_create(
                         job_template=job_template,
-                        status=job_stat, name=job_template.name,
+                        status=job_stat, name="%s-%d" % (job_template.name, job_i),
                         project=job_template.project, inventory=job_template.inventory,
                         credential=job_template.credential,
                         cloud_credential=job_template.cloud_credential,
@@ -626,25 +634,29 @@ try:
                     )
                     job._is_new = _
                     jobs.append(job)
+                    job_i += 1
                     if not job._is_new:
+                        job_template_idx += 1
+                        group_idx += 1
                         continue
-                    if i == n:
+                    if i+1 == n:
                         job_template.last_job = job
                         if job_template.pk % 5 == 0:
                             job_template.current_job = job
                         job_template.save()
 
-                    with transaction.atomic():
+                    if job._is_new:
-                        if job_template.inventory:
+                        with transaction.atomic():
-                            inv_groups = [g for g in job_template.inventory.groups.all()]
+                            if job_template.inventory:
-                            if len(inv_groups):
+                                inv_groups = [g for g in job_template.inventory.groups.all()]
-                                JobHostSummary.objects.bulk_create([
+                                if len(inv_groups):
-                                    JobHostSummary(
+                                    JobHostSummary.objects.bulk_create([
-                                        job=job, host=h, host_name=h.name, processed=1,
+                                        JobHostSummary(
-                                        created=now(), modified=now()
+                                            job=job, host=h, host_name=h.name, processed=1,
-                                    )
+                                            created=now(), modified=now()
-                                    for h in inv_groups[group_idx % len(inv_groups)].hosts.all()[:100]
+                                        )
-                                ])
+                                        for h in inv_groups[group_idx % len(inv_groups)].hosts.all()[:100]
+                                    ])
                     group_idx += 1
                 job_template_idx += 1
                 if n: