Merge pull request #4706 from ryanpetrello/faster-dashboard

optimize dashboard performance for larger UnifiedJob counts

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

awx/api/filters.py

@@ -126,7 +126,7 @@ class FieldLookupBackend(BaseFilterBackend):
     '''
 
     RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by',
-                      'search', 'type', 'host_filter')
+                      'search', 'type', 'host_filter', 'count_disabled',)
 
     SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains',
                          'startswith', 'istartswith', 'endswith', 'iendswith',

awx/api/pagination.py

@@ -3,14 +3,28 @@
 
 # Django REST Framework
 from django.conf import settings
+from django.core.paginator import Paginator as DjangoPaginator
 from rest_framework import pagination
+from rest_framework.response import Response
 from rest_framework.utils.urls import replace_query_param
 
 
+class DisabledPaginator(DjangoPaginator):
+
+    @property
+    def num_pages(self):
+        return 1
+
+    @property
+    def count(self):
+        return 200
+
+
 class Pagination(pagination.PageNumberPagination):
 
     page_size_query_param = 'page_size'
     max_page_size = settings.MAX_PAGE_SIZE
+    count_disabled = False
 
     def get_next_link(self):
         if not self.page.has_next():
@@ -39,3 +53,17 @@ class Pagination(pagination.PageNumberPagination):
                                  for pl in context['page_links']]
 
         return context
+
+    def paginate_queryset(self, queryset, request, **kwargs):
+        self.count_disabled = 'count_disabled' in request.query_params
+        try:
+            if self.count_disabled:
+                self.django_paginator_class = DisabledPaginator
+            return super(Pagination, self).paginate_queryset(queryset, request, **kwargs)
+        finally:
+            self.django_paginator_class = DjangoPaginator
+
+    def get_paginated_response(self, data):
+        if self.count_disabled:
+            return Response({'results': data})
+        return super(Pagination, self).get_paginated_response(data)

awx/api/views/__init__.py

@@ -245,13 +245,6 @@ class DashboardView(APIView):
                                    'total': hg_projects.count(),
                                    'failed': hg_failed_projects.count()}
 
-        user_jobs = get_user_queryset(request.user, models.Job)
-        user_failed_jobs = user_jobs.filter(failed=True)
-        data['jobs'] = {'url': reverse('api:job_list', request=request),
-                        'failure_url': reverse('api:job_list', request=request) + "?failed=True",
-                        'total': user_jobs.count(),
-                        'failed': user_failed_jobs.count()}
-
         user_list = get_user_queryset(request.user, models.User)
         team_list = get_user_queryset(request.user, models.Team)
         credential_list = get_user_queryset(request.user, models.Credential)

awx/main/migrations/0088_v360_dashboard_optimizations.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.4 on 2019-09-10 21:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0087_v360_update_credential_injector_help_text'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='finished',
+            field=models.DateTimeField(db_index=True, default=None, editable=False, help_text='The date and time the job finished execution.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='launch_type',
+            field=models.CharField(choices=[('manual', 'Manual'), ('relaunch', 'Relaunch'), ('callback', 'Callback'), ('scheduled', 'Scheduled'), ('dependency', 'Dependency'), ('workflow', 'Workflow'), ('sync', 'Sync'), ('scm', 'SCM Update')], db_index=True, default='manual', editable=False, max_length=20),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='created',
+            field=models.DateTimeField(db_index=True, default=None, editable=False),
+        ),
+    ]

awx/main/models/unified_jobs.py

@@ -559,11 +559,17 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         related_name='%(class)s_unified_jobs',
         on_delete=polymorphic.SET_NULL,
     )
+    created = models.DateTimeField(
+        default=None,
+        editable=False,
+        db_index=True,  # add an index, this is a commonly queried field
+    )
     launch_type = models.CharField(
         max_length=20,
         choices=LAUNCH_TYPE_CHOICES,
         default='manual',
         editable=False,
+        db_index=True
     )
     schedule = models.ForeignKey( # Which schedule entry was responsible for starting this job.
         'Schedule',
@@ -621,6 +627,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         default=None,
         editable=False,
         help_text=_("The date and time the job finished execution."),
+        db_index=True,
     )
     elapsed = models.DecimalField(
         max_digits=12,

awx/ui/client/src/home/home.controller.js

@@ -85,7 +85,7 @@ export default ['$scope','Wait', '$timeout', 'i18n',
                 ProcessErrors($scope, data, status, null, { hdr: i18n._('Error!'), msg: i18n._(`Failed to get dashboard host graph data: ${status}`) });
             });
 
-            Rest.setUrl(GetBasePath("unified_jobs") + "?order_by=-finished&page_size=5&finished__isnull=false&type=workflow_job,job");
+            Rest.setUrl(GetBasePath("unified_jobs") + "?order_by=-finished&page_size=5&finished__isnull=false&type=workflow_job,job&count_disabled=1");
             Rest.setHeader({'X-WS-Session-Quiet': true});
             Rest.get()
             .then(({data}) => {
@@ -95,7 +95,7 @@ export default ['$scope','Wait', '$timeout', 'i18n',
                 ProcessErrors($scope, data, status, null, { hdr: i18n._('Error!'), msg: i18n._(`Failed to get dashboard jobs list: ${status}`) });
             });
 
-            Rest.setUrl(GetBasePath("unified_job_templates") + "?order_by=-last_job_run&page_size=5&last_job_run__isnull=false&type=workflow_job_template,job_template");
+            Rest.setUrl(GetBasePath("unified_job_templates") + "?order_by=-last_job_run&page_size=5&last_job_run__isnull=false&type=workflow_job_template,job_template&count_disabled=1");
             Rest.get()
             .then(({data}) => {
                 $scope.dashboardJobTemplatesListData = data.results;
@@ -140,7 +140,7 @@ export default ['$scope','Wait', '$timeout', 'i18n',
         .catch(({data, status}) => {
             ProcessErrors($scope, data, status, null, { hdr: i18n._('Error!'), msg: i18n._(`Failed to get dashboard: ${status}`) });
         });
-        Rest.setUrl(GetBasePath("unified_jobs") + "?order_by=-finished&page_size=5&finished__isnull=false&type=workflow_job,job");
+        Rest.setUrl(GetBasePath("unified_jobs") + "?order_by=-finished&page_size=5&finished__isnull=false&type=workflow_job,job&count_disabled=1");
         Rest.setHeader({'X-WS-Session-Quiet': true});
         Rest.get()
         .then(({data}) => {