Revamp user_capabilities with new copy fields

Add copy fields corresponding to new server-side copying

Refactor the way user_capabilities are delivered
 - move the prefetch definition from views to serializer
 - store temporary mapping in serializer context
 - use serializer backlinks to denote polymorphic prefetch model exclusions

awx/api/generics.py

@@ -355,13 +355,6 @@ class ListAPIView(generics.ListAPIView, GenericAPIView):
     def get_queryset(self):
         return self.request.user.get_queryset(self.model)
 
-    def paginate_queryset(self, queryset):
-        page = super(ListAPIView, self).paginate_queryset(queryset)
-        # Queries RBAC info & stores into list objects
-        if hasattr(self, 'capabilities_prefetch') and page is not None:
-            cache_list_capabilities(page, self.capabilities_prefetch, self.model, self.request.user)
-        return page
-
     def get_description_context(self):
         if 'username' in get_all_field_names(self.model):
             order_field = 'username'

awx/api/serializers.py

@@ -48,7 +48,8 @@ from awx.main.fields import ImplicitRoleField
 from awx.main.utils import (
     get_type_for_model, get_model_for_type, timestamp_apiformat,
     camelcase_to_underscore, getattrd, parse_yaml_or_json,
-    has_model_field_prefetched, extract_ansible_vars, encrypt_dict)
+    has_model_field_prefetched, extract_ansible_vars, encrypt_dict,
+    prefetch_page_capabilities)
 from awx.main.utils.filters import SmartFilter
 from awx.main.redact import REPLACE_STR
 
@@ -404,18 +405,47 @@ class BaseSerializer(serializers.ModelSerializer):
 
         # Advance display of RBAC capabilities
         if hasattr(self, 'show_capabilities'):
-            view = self.context.get('view', None)
+            user_capabilities = self._obj_capability_dict(obj)
-            parent_obj = None
+            if user_capabilities:
-            if view and hasattr(view, 'parent_model') and hasattr(view, 'get_parent_object'):
+                summary_fields['user_capabilities'] = user_capabilities
-                parent_obj = view.get_parent_object()
-            if view and view.request and view.request.user:
-                user_capabilities = get_user_capabilities(
-                    view.request.user, obj, method_list=self.show_capabilities, parent_obj=parent_obj)
-                if user_capabilities:
-                    summary_fields['user_capabilities'] = user_capabilities
 
         return summary_fields
 
+    def _obj_capability_dict(self, obj):
+        """
+        Returns the user_capabilities dictionary for a single item
+        If inside of a list view, it runs the prefetching algorithm for
+        the entire current page, saves it into context
+        """
+        view = self.context.get('view', None)
+        parent_obj = None
+        if view and hasattr(view, 'parent_model') and hasattr(view, 'get_parent_object'):
+            parent_obj = view.get_parent_object()
+        if view and view.request and view.request.user:
+            capabilities_cache = {}
+            # if serializer has parent, it is ListView, apply page capabilities prefetch
+            if self.parent and hasattr(self, 'capabilities_prefetch') and self.capabilities_prefetch:
+                qs = self.parent.instance
+                if 'capability_map' not in self.context:
+                    if hasattr(self, 'polymorphic_base'):
+                        model = self.polymorphic_base.Meta.model
+                        prefetch_list = self.polymorphic_base.capabilities_prefetch
+                    else:
+                        model = self.Meta.model
+                        prefetch_list = self.capabilities_prefetch
+                    self.context['capability_map'] = prefetch_page_capabilities(
+                        model, qs, prefetch_list, view.request.user
+                    )
+                if obj.id in self.context['capability_map']:
+                    capabilities_cache = self.context['capability_map'][obj.id]
+            return get_user_capabilities(
+                view.request.user, obj, method_list=self.show_capabilities, parent_obj=parent_obj,
+                capabilities_cache=capabilities_cache
+            )
+        else:
+            # Contextual information to produce user_capabilities doesn't exist
+            return {}
+
     def get_created(self, obj):
         if obj is None:
             return None
@@ -600,6 +630,11 @@ class BaseFactSerializer(BaseSerializer):
 
 
 class UnifiedJobTemplateSerializer(BaseSerializer):
+    capabilities_prefetch = [
+        'admin', 'execute',
+        {'copy': ['jobtemplate.project.use', 'jobtemplate.inventory.use',
+                  'workflowjobtemplate.organization.workflow_admin']}
+    ]
 
     class Meta:
         model = UnifiedJobTemplate
@@ -637,6 +672,13 @@ class UnifiedJobTemplateSerializer(BaseSerializer):
                 serializer_class = WorkflowJobTemplateSerializer
         if serializer_class:
             serializer = serializer_class(instance=obj, context=self.context)
+            # preserve links for list view
+            if self.parent:
+                serializer.parent = self.parent
+                serializer.polymorphic_base = self
+                # Exclude certain models from capabilities prefetch
+                if isinstance(obj, (Project, InventorySource, SystemJobTemplate)):
+                    obj.capabilities_prefetch = None
             return serializer.to_representation(obj)
         else:
             return super(UnifiedJobTemplateSerializer, self).to_representation(obj)
@@ -1305,7 +1347,11 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
     status = serializers.ChoiceField(choices=Project.PROJECT_STATUS_CHOICES, read_only=True)
     last_update_failed = serializers.BooleanField(read_only=True)
     last_updated = serializers.DateTimeField(read_only=True)
-    show_capabilities = ['start', 'schedule', 'edit', 'delete']
+    show_capabilities = ['start', 'schedule', 'edit', 'delete', 'copy']
+    capabilities_prefetch = [
+        'admin', 'update',
+        {'copy': 'organization.project_admin'}
+    ]
 
     class Meta:
         model = Project
@@ -1457,7 +1503,11 @@ class BaseSerializerWithVariables(BaseSerializer):
 
 
 class InventorySerializer(BaseSerializerWithVariables):
-    show_capabilities = ['edit', 'delete', 'adhoc']
+    show_capabilities = ['edit', 'delete', 'adhoc', 'copy']
+    capabilities_prefetch = [
+        'admin', 'adhoc',
+        {'copy': 'organization.inventory_admin'}
+    ]
 
     class Meta:
         model = Inventory
@@ -1547,6 +1597,7 @@ class InventoryScriptSerializer(InventorySerializer):
 
 class HostSerializer(BaseSerializerWithVariables):
     show_capabilities = ['edit', 'delete']
+    capabilities_prefetch = ['inventory.admin']
 
     class Meta:
         model = Host
@@ -1676,6 +1727,7 @@ class AnsibleFactsSerializer(BaseSerializer):
 
 
 class GroupSerializer(BaseSerializerWithVariables):
+    capabilities_prefetch = ['inventory.admin', 'inventory.adhoc']
 
     class Meta:
         model = Group
@@ -1815,7 +1867,10 @@ class GroupVariableDataSerializer(BaseVariableDataSerializer):
 class CustomInventoryScriptSerializer(BaseSerializer):
 
     script = serializers.CharField(trim_whitespace=False)
-    show_capabilities = ['edit', 'delete']
+    show_capabilities = ['edit', 'delete', 'copy']
+    capabilities_prefetch = [
+        {'edit': 'organization.admin'}
+    ]
 
     class Meta:
         model = CustomInventoryScript
@@ -2431,7 +2486,8 @@ class V2CredentialFields(BaseSerializer):
 
 
 class CredentialSerializer(BaseSerializer):
-    show_capabilities = ['edit', 'delete']
+    show_capabilities = ['edit', 'delete', 'copy']
+    capabilities_prefetch = ['admin', 'use']
 
     class Meta:
         model = Credential
@@ -2926,6 +2982,10 @@ class JobTemplateMixin(object):
 
 class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobOptionsSerializer):
     show_capabilities = ['start', 'schedule', 'copy', 'edit', 'delete']
+    capabilities_prefetch = [
+        'admin', 'execute',
+        {'copy': ['project.use', 'inventory.use']}
+    ]
 
     status = serializers.ChoiceField(choices=JobTemplate.JOB_TEMPLATE_STATUS_CHOICES, read_only=True, required=False)
 
@@ -3389,6 +3449,10 @@ class SystemJobCancelSerializer(SystemJobSerializer):
 
 class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJobTemplateSerializer):
     show_capabilities = ['start', 'schedule', 'edit', 'copy', 'delete']
+    capabilities_prefetch = [
+        'admin', 'execute',
+        {'copy': 'organization.workflow_admin'}
+    ]
 
     class Meta:
         model = WorkflowJobTemplate
@@ -4203,7 +4267,8 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
 
 
 class NotificationTemplateSerializer(BaseSerializer):
-    show_capabilities = ['edit', 'delete']
+    show_capabilities = ['edit', 'delete', 'copy']
+    capabilities_prefetch = [{'copy': 'organization.admin'}]
 
     class Meta:
         model = NotificationTemplate

awx/api/views.py

@@ -1205,7 +1205,6 @@ class ProjectList(ListCreateAPIView):
 
     model = Project
     serializer_class = ProjectSerializer
-    capabilities_prefetch = ['admin', 'update']
 
     def get_queryset(self):
         projects_qs = Project.accessible_objects(self.request.user, 'read_role')
@@ -1814,7 +1813,6 @@ class CredentialList(CredentialViewMixin, ListCreateAPIView):
 
     model = Credential
     serializer_class = CredentialSerializerCreate
-    capabilities_prefetch = ['admin', 'use']
     filter_backends = ListCreateAPIView.filter_backends + [V1CredentialFilterBackend]
 
 
@@ -1981,7 +1979,6 @@ class InventoryList(ListCreateAPIView):
 
     model = Inventory
     serializer_class = InventorySerializer
-    capabilities_prefetch = ['admin', 'adhoc']
 
     def get_queryset(self):
         qs = Inventory.accessible_objects(self.request.user, 'read_role')
@@ -2120,7 +2117,6 @@ class HostList(HostRelatedSearchMixin, ListCreateAPIView):
     always_allow_superuser = False
     model = Host
     serializer_class = HostSerializer
-    capabilities_prefetch = ['inventory.admin']
 
     def get_queryset(self):
         qs = super(HostList, self).get_queryset()
@@ -2157,7 +2153,6 @@ class InventoryHostsList(HostRelatedSearchMixin, SubListCreateAttachDetachAPIVie
     parent_model = Inventory
     relationship = 'hosts'
     parent_key = 'inventory'
-    capabilities_prefetch = ['inventory.admin']
 
     def get_queryset(self):
         inventory = self.get_parent_object()
@@ -2334,7 +2329,6 @@ class GroupList(ListCreateAPIView):
 
     model = Group
     serializer_class = GroupSerializer
-    capabilities_prefetch = ['inventory.admin', 'inventory.adhoc']
 
 
 class EnforceParentRelationshipMixin(object):
@@ -2421,7 +2415,6 @@ class GroupHostsList(HostRelatedSearchMixin,
     serializer_class = HostSerializer
     parent_model = Group
     relationship = 'hosts'
-    capabilities_prefetch = ['inventory.admin']
 
     def update_raw_data(self, data):
         data.pop('inventory', None)
@@ -2448,7 +2441,6 @@ class GroupAllHostsList(HostRelatedSearchMixin, SubListAPIView):
     serializer_class = HostSerializer
     parent_model = Group
     relationship = 'hosts'
-    capabilities_prefetch = ['inventory.admin']
 
     def get_queryset(self):
         parent = self.get_parent_object()
@@ -2747,7 +2739,6 @@ class InventorySourceHostsList(HostRelatedSearchMixin, SubListDestroyAPIView):
     parent_model = InventorySource
     relationship = 'hosts'
     check_sub_obj_permission = False
-    capabilities_prefetch = ['inventory.admin']
 
 
 class InventorySourceGroupsList(SubListDestroyAPIView):
@@ -2860,10 +2851,6 @@ class JobTemplateList(ListCreateAPIView):
     metadata_class = JobTypeMetadata
     serializer_class = JobTemplateSerializer
     always_allow_superuser = False
-    capabilities_prefetch = [
-        'admin', 'execute',
-        {'copy': ['project.use', 'inventory.use']}
-    ]
 
     def post(self, request, *args, **kwargs):
         ret = super(JobTemplateList, self).post(request, *args, **kwargs)
@@ -4276,7 +4263,6 @@ class JobEventHostsList(HostRelatedSearchMixin, SubListAPIView):
     parent_model = JobEvent
     relationship = 'hosts'
     view_name = _('Job Event Hosts List')
-    capabilities_prefetch = ['inventory.admin']
 
 
 class BaseJobEventsList(SubListAPIView):
@@ -4570,11 +4556,6 @@ class UnifiedJobTemplateList(ListAPIView):
 
     model = UnifiedJobTemplate
     serializer_class = UnifiedJobTemplateSerializer
-    capabilities_prefetch = [
-        'admin', 'execute',
-        {'copy': ['jobtemplate.project.use', 'jobtemplate.inventory.use',
-                  'workflowjobtemplate.organization.workflow_admin']}
-    ]
 
 
 class UnifiedJobList(ListAPIView):

awx/main/access.py

@@ -238,6 +238,9 @@ class BaseAccess(object):
     def can_delete(self, obj):
         return self.user.is_superuser
 
+    def can_copy(self, obj):
+        return self.can_add({'reference_obj': obj})
+
     def can_attach(self, obj, sub_obj, relationship, data,
                    skip_sub_obj_read_check=False):
         if skip_sub_obj_read_check:
@@ -333,7 +336,7 @@ class BaseAccess(object):
             elif "features" not in validation_info:
                 raise LicenseForbids(_("Features not found in active license."))
 
-    def get_user_capabilities(self, obj, method_list=[], parent_obj=None):
+    def get_user_capabilities(self, obj, method_list=[], parent_obj=None, capabilities_cache={}):
         if obj is None:
             return {}
         user_capabilities = {}
@@ -356,6 +359,10 @@ class BaseAccess(object):
             elif display_method == 'copy' and isinstance(obj, WorkflowJobTemplate) and obj.organization_id is None:
                 user_capabilities[display_method] = self.user.is_superuser
                 continue
+            elif display_method == 'copy' and isinstance(obj, Project) and obj.scm_type == '':
+                # Connot copy manual project without errors
+                user_capabilities[display_method] = False
+                continue
             elif display_method in ['start', 'schedule'] and isinstance(obj, Group):  # TODO: remove in 3.3
                 try:
                     if obj.deprecated_inventory_source and not obj.deprecated_inventory_source._can_update():
@@ -370,8 +377,8 @@ class BaseAccess(object):
                     continue
 
             # Grab the answer from the cache, if available
-            if hasattr(obj, 'capabilities_cache') and display_method in obj.capabilities_cache:
+            if display_method in capabilities_cache:
-                user_capabilities[display_method] = obj.capabilities_cache[display_method]
+                user_capabilities[display_method] = capabilities_cache[display_method]
                 if self.user.is_superuser and not user_capabilities[display_method]:
                     # Cache override for models with bad orphaned state
                     user_capabilities[display_method] = True
@@ -389,10 +396,10 @@ class BaseAccess(object):
             if display_method == 'schedule':
                 user_capabilities['schedule'] = user_capabilities['start']
                 continue
-            elif display_method == 'delete' and not isinstance(obj, (User, UnifiedJob)):
+            elif display_method == 'delete' and not isinstance(obj, (User, UnifiedJob, CustomInventoryScript)):
                 user_capabilities['delete'] = user_capabilities['edit']
                 continue
-            elif display_method == 'copy' and isinstance(obj, (Group, Host)):
+            elif display_method == 'copy' and isinstance(obj, (Group, Host, CustomInventoryScript)):
                 user_capabilities['copy'] = user_capabilities['edit']
                 continue
 
@@ -1316,9 +1323,6 @@ class JobTemplateAccess(BaseAccess):
         else:
             return False
 
-    def can_copy(self, obj):
-        return self.can_add({'reference_obj': obj})
-
     def can_start(self, obj, validate_license=True):
         # Check license.
         if validate_license:

awx/main/models/unified_jobs.py

@@ -180,12 +180,6 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio
         else:
             return super(UnifiedJobTemplate, self).unique_error_message(model_class, unique_check)
 
-    @classmethod
-    def invalid_user_capabilities_prefetch_models(cls):
-        if cls != UnifiedJobTemplate:
-            return []
-        return ['project', 'inventorysource', 'systemjobtemplate']
-
     @classmethod
     def _submodels_with_roles(cls):
         ujt_classes = [c for c in cls.__subclasses__()

awx/main/utils/common.py

@@ -32,6 +32,7 @@ from django.db import DatabaseError
 from django.utils.translation import ugettext_lazy as _
 from django.db.models.fields.related import ForeignObjectRel, ManyToManyField
 from django.db.models.query import QuerySet
+from django.db.models import Q
 
 # Django REST Framework
 from rest_framework.exceptions import ParseError, PermissionDenied
@@ -44,7 +45,7 @@ logger = logging.getLogger('awx.main.utils')
 __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore', 'memoize', 'memoize_delete',
            'get_ansible_version', 'get_ssh_version', 'get_licenser', 'get_awx_version', 'update_scm_url',
            'get_type_for_model', 'get_model_for_type', 'copy_model_by_class',
-           'copy_m2m_relationships', 'cache_list_capabilities', 'to_python_boolean',
+           'copy_m2m_relationships', 'prefetch_page_capabilities', 'to_python_boolean',
            'ignore_inventory_computed_fields', 'ignore_inventory_group_removal',
            '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'NoDefaultProvided',
            'get_current_apps', 'set_current_apps', 'OutputEventFilter',
@@ -503,7 +504,6 @@ def get_model_for_type(type):
     '''
     Return model class for a given type name.
     '''
-    from django.db.models import Q
     from django.contrib.contenttypes.models import ContentType
     for ct in ContentType.objects.filter(Q(app_label='main') | Q(app_label='auth', model='user')):
         ct_model = ct.model_class()
@@ -516,30 +516,31 @@ def get_model_for_type(type):
         raise DatabaseError('"{}" is not a valid AWX model.'.format(type))
 
 
-def cache_list_capabilities(page, prefetch_list, model, user):
+def prefetch_page_capabilities(model, page, prefetch_list, user):
     '''
-    Given a `page` list of objects, the specified roles for the specified user
+    Given a `page` list of objects, a nested dictionary of user_capabilities
-    are save on each object in the list, using 1 query for each role type
+    are returned by id, ex.
+    {
+        4: {'edit': True, 'start': True},
+        6: {'edit': False, 'start': False}
+    }
+    Each capability is produced for all items in the page in a single query
 
-    Examples:
+    Examples of prefetch language:
-    capabilities_prefetch = ['admin', 'execute']
+    prefetch_list = ['admin', 'execute']
       --> prefetch the admin (edit) and execute (start) permissions for
           items in list for current user
-    capabilities_prefetch = ['inventory.admin']
+    prefetch_list = ['inventory.admin']
       --> prefetch the related inventory FK permissions for current user,
           and put it into the object's cache
-    capabilities_prefetch = [{'copy': ['inventory.admin', 'project.admin']}]
+    prefetch_list = [{'copy': ['inventory.admin', 'project.admin']}]
       --> prefetch logical combination of admin permission to inventory AND
           project, put into cache dictionary as "copy"
     '''
-    from django.db.models import Q
     page_ids = [obj.id for obj in page]
+    mapping = {}
     for obj in page:
-        obj.capabilities_cache = {}
+        mapping[obj.id] = {}
-
-    skip_models = []
-    if hasattr(model, 'invalid_user_capabilities_prefetch_models'):
-        skip_models = model.invalid_user_capabilities_prefetch_models()
 
     for prefetch_entry in prefetch_list:
 
@@ -583,11 +584,9 @@ def cache_list_capabilities(page, prefetch_list, model, user):
 
         # Save data item-by-item
         for obj in page:
-            if skip_models and obj.__class__.__name__.lower() in skip_models:
+            mapping[obj.pk][display_method] = bool(obj.pk in ids_with_role)
-                continue
+
-            obj.capabilities_cache[display_method] = False
+    return mapping
-            if obj.pk in ids_with_role:
-                obj.capabilities_cache[display_method] = True
 
 
 def validate_vars_type(vars_obj):