From 28ef16e1e8cbdb80fa932aafcaff057140552c97 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 24 Jan 2017 10:14:44 -0500 Subject: [PATCH 1/4] activitiy stream optimization for org admins with large data sets --- awx/main/access.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index aeeef9fa54..db27608af8 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -2102,11 +2102,10 @@ class ActivityStreamAccess(BaseAccess): - custom inventory scripts ''' qs = self.model.objects.all() - qs = qs.select_related('actor') qs = qs.prefetch_related('organization', 'user', 'inventory', 'host', 'group', 'inventory_source', 'inventory_update', 'credential', 'team', 'project', 'project_update', 'permission', 'job_template', 'job', 'ad_hoc_command', - 'notification_template', 'notification', 'label', 'role') + 'notification_template', 'notification', 'label', 'role', 'actor') if self.user.is_superuser or self.user.is_system_auditor: return qs.all() From eaffde438ba83fee0032ba5a9f354b1783d14e82 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 24 Jan 2017 10:15:33 -0500 Subject: [PATCH 2/4] encode logic for JT name-uniqueness in data generator --- tools/data_generators/rbac_dummy_data_generator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py index 4d0a319f5a..5907e7a34a 100755 --- a/tools/data_generators/rbac_dummy_data_generator.py +++ b/tools/data_generators/rbac_dummy_data_generator.py @@ -479,14 +479,14 @@ try: job_template, _ = JobTemplate.objects.get_or_create( name='%s Job Template %d Project %d' % (prefix, job_template_id, project_idx), - inventory=inventory, - project=project, - credential=next(credential_gen), defaults=dict( + inventory=inventory, + project=project, + credential=next(credential_gen), created_by=next(creator_gen), modified_by=next(modifier_gen), - playbook="debug.yml"), - **extra_kwargs + playbook="debug.yml", + **extra_kwargs) ) job_template._is_new = _ job_templates.append(job_template) From 77f0625d442e2a748e30a4dab361389b3db0d930 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 24 Jan 2017 12:45:44 -0500 Subject: [PATCH 3/4] include WFJTs in activity stream for org_admins --- awx/api/serializers.py | 8 ++++---- awx/main/access.py | 8 +++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 1b642e63dc..119283d181 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2999,10 +2999,10 @@ class ActivityStreamSerializer(BaseSerializer): rel = {} if obj.actor is not None: rel['actor'] = reverse('api:user_detail', args=(obj.actor.pk,)) - for fk, __ in SUMMARIZABLE_FK_FIELDS.items(): + for fk, __ in SUMMARIZABLE_FK_FIELDS.items() + [('workflow_job_template', 0)]: if not hasattr(obj, fk): continue - allm2m = getattr(obj, fk).distinct() + allm2m = getattr(obj, fk).all() if getattr(obj, fk).exists(): rel[fk] = [] for thisItem in allm2m: @@ -3017,11 +3017,11 @@ class ActivityStreamSerializer(BaseSerializer): def get_summary_fields(self, obj): summary_fields = OrderedDict() - for fk, related_fields in SUMMARIZABLE_FK_FIELDS.items(): + for fk, related_fields in SUMMARIZABLE_FK_FIELDS.items() + [('workflow_job_template', DEFAULT_SUMMARY_FIELDS)]: try: if not hasattr(obj, fk): continue - allm2m = getattr(obj, fk).distinct() + allm2m = getattr(obj, fk).all() if getattr(obj, fk).exists(): summary_fields[fk] = [] for thisItem in allm2m: diff --git a/awx/main/access.py b/awx/main/access.py index db27608af8..42314bc9b9 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -2105,7 +2105,9 @@ class ActivityStreamAccess(BaseAccess): qs = qs.prefetch_related('organization', 'user', 'inventory', 'host', 'group', 'inventory_source', 'inventory_update', 'credential', 'team', 'project', 'project_update', 'permission', 'job_template', 'job', 'ad_hoc_command', - 'notification_template', 'notification', 'label', 'role', 'actor') + 'notification_template', 'notification', 'label', 'role', 'actor', + 'schedule', 'role', 'custom_inventory_script', 'unified_job_template', + 'workflow_job_template') if self.user.is_superuser or self.user.is_system_auditor: return qs.all() @@ -2118,6 +2120,7 @@ class ActivityStreamAccess(BaseAccess): project_set = Project.accessible_objects(self.user, 'read_role') jt_set = JobTemplate.accessible_objects(self.user, 'read_role') team_set = Team.accessible_objects(self.user, 'read_role') + wfjt_set = WorkflowJobTemplate.accessible_objects(self.user, 'read_role') return qs.filter( Q(ad_hoc_command__inventory__in=inventory_set) | @@ -2135,6 +2138,9 @@ class ActivityStreamAccess(BaseAccess): Q(project_update__project__in=project_set) | Q(job_template__in=jt_set) | Q(job__job_template__in=jt_set) | + Q(workflow_job_template__in=wfjt_set) | + Q(workflow_job_template_node__workflow_job_template__in=wfjt_set) | + Q(workflow_job__workflow_job_template__in=wfjt_set) | Q(notification_template__organization__in=auditing_orgs) | Q(notification__notification_template__organization__in=auditing_orgs) | Q(label__organization__in=auditing_orgs) | From 3b4af0aa820e313fa8a8f336271c176c8f68bdf8 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 26 Jan 2017 11:23:47 -0500 Subject: [PATCH 4/4] WFJT activity stream optimization --- awx/api/serializers.py | 13 +++++++++++-- awx/api/views.py | 7 +++++++ awx/main/access.py | 6 +++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 119283d181..b87b711404 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -83,6 +83,8 @@ SUMMARIZABLE_FK_FIELDS = { 'network_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'net'), 'job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'elapsed'), 'job_template': DEFAULT_SUMMARY_FIELDS, + 'workflow_job_template': DEFAULT_SUMMARY_FIELDS, + 'workflow_job': DEFAULT_SUMMARY_FIELDS, 'schedule': DEFAULT_SUMMARY_FIELDS + ('next_run',), 'unified_job_template': DEFAULT_SUMMARY_FIELDS + ('unified_job_type',), 'last_job': DEFAULT_SUMMARY_FIELDS + ('finished', 'status', 'failed', 'license_error'), @@ -2999,13 +3001,17 @@ class ActivityStreamSerializer(BaseSerializer): rel = {} if obj.actor is not None: rel['actor'] = reverse('api:user_detail', args=(obj.actor.pk,)) - for fk, __ in SUMMARIZABLE_FK_FIELDS.items() + [('workflow_job_template', 0)]: + for fk, __ in SUMMARIZABLE_FK_FIELDS.items(): if not hasattr(obj, fk): continue allm2m = getattr(obj, fk).all() if getattr(obj, fk).exists(): rel[fk] = [] + id_list = [] for thisItem in allm2m: + if getattr(thisItem, 'id', None) in id_list: + continue + id_list.append(getattr(thisItem, 'id', None)) if fk == 'custom_inventory_script': rel[fk].append(reverse('api:inventory_script_detail', args=(thisItem.id,))) else: @@ -3017,7 +3023,7 @@ class ActivityStreamSerializer(BaseSerializer): def get_summary_fields(self, obj): summary_fields = OrderedDict() - for fk, related_fields in SUMMARIZABLE_FK_FIELDS.items() + [('workflow_job_template', DEFAULT_SUMMARY_FIELDS)]: + for fk, related_fields in SUMMARIZABLE_FK_FIELDS.items(): try: if not hasattr(obj, fk): continue @@ -3050,6 +3056,9 @@ class ActivityStreamSerializer(BaseSerializer): thisItemDict[field] = fval if fk == 'group': thisItemDict['inventory_id'] = getattr(thisItem, 'inventory_id', None) + if thisItemDict.get('id', None): + if thisItemDict.get('id', None) in [obj_dict.get('id', None) for obj_dict in summary_fields[fk]]: + continue summary_fields[fk].append(thisItemDict) except ObjectDoesNotExist: pass diff --git a/awx/api/views.py b/awx/api/views.py index d65ca3c478..8f1b0ce70a 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -3087,6 +3087,13 @@ class WorkflowJobTemplateActivityStreamList(WorkflowsEnforcementMixin, ActivityS relationship = 'activitystream_set' new_in_310 = True + def get_queryset(self): + parent = self.get_parent_object() + self.check_parent_access(parent) + qs = self.request.user.get_queryset(self.model) + return qs.filter(Q(workflow_job_template=parent) | + Q(workflow_job_template_node__workflow_job_template=parent)) + class WorkflowJobList(WorkflowsEnforcementMixin, ListCreateAPIView): diff --git a/awx/main/access.py b/awx/main/access.py index 42314bc9b9..2ca8163e98 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -2104,10 +2104,10 @@ class ActivityStreamAccess(BaseAccess): qs = self.model.objects.all() qs = qs.prefetch_related('organization', 'user', 'inventory', 'host', 'group', 'inventory_source', 'inventory_update', 'credential', 'team', 'project', 'project_update', - 'permission', 'job_template', 'job', 'ad_hoc_command', + 'job_template', 'job', 'ad_hoc_command', 'notification_template', 'notification', 'label', 'role', 'actor', - 'schedule', 'role', 'custom_inventory_script', 'unified_job_template', - 'workflow_job_template') + 'schedule', 'custom_inventory_script', 'unified_job_template', + 'workflow_job_template', 'workflow_job') if self.user.is_superuser or self.user.is_system_auditor: return qs.all()