From 2131703ca09832dc252bfaa3715c2f17bbfed750 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 20 May 2021 21:07:41 -0400 Subject: [PATCH] add/remove indexes, more get_event_querset() * Do not cascade delete unified job events. We will clean those up in cleanup_job runs * Add limit pagination to all unified job events endpoints --- awx/api/pagination.py | 2 +- awx/api/serializers.py | 8 +- awx/api/views/__init__.py | 9 +- awx/api/views/inventory.py | 4 + awx/main/migrations/0144_event_partitions.py | 132 +++++++++++++++++-- awx/main/models/events.py | 78 +++++++---- 6 files changed, 190 insertions(+), 43 deletions(-) diff --git a/awx/api/pagination.py b/awx/api/pagination.py index 452ef7443f..68db8cceab 100644 --- a/awx/api/pagination.py +++ b/awx/api/pagination.py @@ -103,7 +103,7 @@ class LimitPagination(pagination.BasePagination): return self.default_limit -class JobEventPagination(Pagination): +class UnifiedJobEventPagination(Pagination): """ By default, use Pagination for all operations. If `limit` query parameter specified use LimitPagination diff --git a/awx/api/serializers.py b/awx/api/serializers.py index f91ebad8f5..dc50b72237 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -3044,7 +3044,7 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer): res = super(JobSerializer, self).get_related(obj) res.update( dict( - job_events=self.reverse('api:job_job_events_list', kwargs={'pk': obj.pk}), + job_events=self.reverse('api:job_job_events_list', kwargs={'pk': obj.pk}), # TODO: consider adding job_created job_host_summaries=self.reverse('api:job_job_host_summaries_list', kwargs={'pk': obj.pk}), activity_stream=self.reverse('api:job_activity_stream_list', kwargs={'pk': obj.pk}), notifications=self.reverse('api:job_notifications_list', kwargs={'pk': obj.pk}), @@ -3111,8 +3111,8 @@ class JobDetailSerializer(JobSerializer): fields = ('*', 'host_status_counts', 'playbook_counts', 'custom_virtualenv') def get_playbook_counts(self, obj): - task_count = obj.job_events.filter(event='playbook_on_task_start').count() - play_count = obj.job_events.filter(event='playbook_on_play_start').count() + task_count = obj.get_event_queryset().filter(event='playbook_on_task_start').count() + play_count = obj.get_event_queryset().filter(event='playbook_on_play_start').count() data = {'play_count': play_count, 'task_count': task_count} @@ -3120,7 +3120,7 @@ class JobDetailSerializer(JobSerializer): def get_host_status_counts(self, obj): try: - counts = obj.job_events.only('event_data').get(event='playbook_on_stats').get_host_status_counts() + counts = obj.get_event_queryset().only('event_data').get(event='playbook_on_stats').get_host_status_counts() except JobEvent.DoesNotExist: counts = {} diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 82aacdbdc9..7a6ced3a91 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -172,7 +172,7 @@ from awx.api.views.root import ( # noqa ApiV2AttachView, ) from awx.api.views.webhooks import WebhookKeyView, GithubWebhookReceiver, GitlabWebhookReceiver # noqa -from awx.api.pagination import JobEventPagination +from awx.api.pagination import UnifiedJobEventPagination logger = logging.getLogger('awx.api.views') @@ -888,6 +888,7 @@ class ProjectUpdateEventsList(SubListAPIView): relationship = 'project_update_events' name = _('Project Update Events List') search_fields = ('stdout',) + pagination_class = UnifiedJobEventPagination def finalize_response(self, request, response, *args, **kwargs): response['X-UI-Max-Events'] = settings.MAX_UI_JOB_EVENTS @@ -907,6 +908,7 @@ class SystemJobEventsList(SubListAPIView): relationship = 'system_job_events' name = _('System Job Events List') search_fields = ('stdout',) + pagination_class = UnifiedJobEventPagination def finalize_response(self, request, response, *args, **kwargs): response['X-UI-Max-Events'] = settings.MAX_UI_JOB_EVENTS @@ -3622,7 +3624,7 @@ class JobRelaunch(RetrieveAPIView): status=status.HTTP_400_BAD_REQUEST, ) host_qs = obj.retry_qs(retry_hosts) - if not obj.job_events.filter(event='playbook_on_stats').exists(): + if not obj.get_event_queryset().filter(event='playbook_on_stats').exists(): return Response( {'hosts': _('Cannot retry on {status_value} hosts, playbook stats not available.').format(status_value=retry_hosts)}, status=status.HTTP_400_BAD_REQUEST, @@ -3833,7 +3835,7 @@ class GroupJobEventsList(BaseJobEventsList): class JobJobEventsList(BaseJobEventsList): parent_model = models.Job - pagination_class = JobEventPagination + pagination_class = UnifiedJobEventPagination def get_queryset(self): job = self.get_parent_object() @@ -4021,6 +4023,7 @@ class BaseAdHocCommandEventsList(NoTruncateMixin, SubListAPIView): relationship = 'ad_hoc_command_events' name = _('Ad Hoc Command Events List') search_fields = ('stdout',) + pagination_class = UnifiedJobEventPagination def get_queryset(self): parent = self.get_parent_object() diff --git a/awx/api/views/inventory.py b/awx/api/views/inventory.py index f179424ccc..7a46ce3511 100644 --- a/awx/api/views/inventory.py +++ b/awx/api/views/inventory.py @@ -38,6 +38,9 @@ from awx.api.serializers import ( ) from awx.api.views.mixin import RelatedJobsPreventDeleteMixin, ControlledByScmMixin +from awx.api.pagination import UnifiedJobEventPagination + + logger = logging.getLogger('awx.api.views.organization') @@ -49,6 +52,7 @@ class InventoryUpdateEventsList(SubListAPIView): relationship = 'inventory_update_events' name = _('Inventory Update Events List') search_fields = ('stdout',) + pagination_class = UnifiedJobEventPagination def get_queryset(self): iu = self.get_parent_object() diff --git a/awx/main/migrations/0144_event_partitions.py b/awx/main/migrations/0144_event_partitions.py index 774508e632..59aa1e5253 100644 --- a/awx/main/migrations/0144_event_partitions.py +++ b/awx/main/migrations/0144_event_partitions.py @@ -47,12 +47,17 @@ def migrate_event_data(apps, schema_editor): cursor.execute(f'DROP TABLE tmp_{tblname}') - # let's go ahead and add and subtract a few indexes while we're here - cursor.execute(f'CREATE INDEX {tblname}_modified_idx ON {tblname} (modified);') - # recreate primary key constraint cursor.execute(f'ALTER TABLE ONLY {tblname} ' f'ADD CONSTRAINT {tblname}_pkey_new PRIMARY KEY (id, job_created);') + with connection.cursor() as cursor: + """ + Big int migration introduced the brin index main_jobevent_job_id_brin_idx index. For upgardes, we drop the index, new installs do nothing. + I have seen the second index in my dev environment. I can not find where in the code it was created. Drop it just in case + """ + cursor.execute('DROP INDEX IF EXISTS main_jobevent_job_id_brin_idx') + cursor.execute('DROP INDEX IF EXISTS main_jobevent_job_id_idx') + class FakeAddField(migrations.AddField): def database_forwards(self, *args): @@ -94,11 +99,6 @@ class Migration(migrations.Migration): name='job_created', field=models.DateTimeField(null=True, editable=False), ), - migrations.AlterField( - model_name='jobevent', - name='job', - field=models.ForeignKey(editable=False, null=True, on_delete=models.deletion.DO_NOTHING, related_name='job_events', to='main.Job'), - ), migrations.CreateModel( name='UnpartitionedAdHocCommandEvent', fields=[], @@ -149,4 +149,120 @@ class Migration(migrations.Migration): }, bases=('main.systemjobevent',), ), + migrations.AlterField( + model_name='adhoccommandevent', + name='ad_hoc_command', + field=models.ForeignKey( + db_index=False, editable=False, on_delete=models.deletion.DO_NOTHING, related_name='ad_hoc_command_events', to='main.AdHocCommand' + ), + ), + migrations.AlterField( + model_name='adhoccommandevent', + name='created', + field=models.DateTimeField(default=None, editable=False, null=True), + ), + migrations.AlterField( + model_name='adhoccommandevent', + name='modified', + field=models.DateTimeField(db_index=True, default=None, editable=False), + ), + migrations.AlterField( + model_name='inventoryupdateevent', + name='created', + field=models.DateTimeField(default=None, editable=False, null=True), + ), + migrations.AlterField( + model_name='inventoryupdateevent', + name='inventory_update', + field=models.ForeignKey( + db_index=False, editable=False, on_delete=models.deletion.DO_NOTHING, related_name='inventory_update_events', to='main.InventoryUpdate' + ), + ), + migrations.AlterField( + model_name='inventoryupdateevent', + name='modified', + field=models.DateTimeField(db_index=True, default=None, editable=False), + ), + migrations.AlterField( + model_name='jobevent', + name='created', + field=models.DateTimeField(default=None, editable=False, null=True), + ), + migrations.AlterField( + model_name='jobevent', + name='job', + field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=models.deletion.DO_NOTHING, related_name='job_events', to='main.Job'), + ), + migrations.AlterField( + model_name='jobevent', + name='modified', + field=models.DateTimeField(db_index=True, default=None, editable=False), + ), + migrations.AlterField( + model_name='projectupdateevent', + name='created', + field=models.DateTimeField(default=None, editable=False, null=True), + ), + migrations.AlterField( + model_name='projectupdateevent', + name='modified', + field=models.DateTimeField(db_index=True, default=None, editable=False), + ), + migrations.AlterField( + model_name='projectupdateevent', + name='project_update', + field=models.ForeignKey( + db_index=False, editable=False, on_delete=models.deletion.DO_NOTHING, related_name='project_update_events', to='main.ProjectUpdate' + ), + ), + migrations.AlterField( + model_name='systemjobevent', + name='created', + field=models.DateTimeField(default=None, editable=False, null=True), + ), + migrations.AlterField( + model_name='systemjobevent', + name='modified', + field=models.DateTimeField(db_index=True, default=None, editable=False), + ), + migrations.AlterField( + model_name='systemjobevent', + name='system_job', + field=models.ForeignKey( + db_index=False, editable=False, on_delete=models.deletion.DO_NOTHING, related_name='system_job_events', to='main.SystemJob' + ), + ), + migrations.AlterIndexTogether( + name='adhoccommandevent', + index_together={ + ('ad_hoc_command', 'job_created', 'event'), + ('ad_hoc_command', 'job_created', 'counter'), + ('ad_hoc_command', 'job_created', 'uuid'), + }, + ), + migrations.AlterIndexTogether( + name='inventoryupdateevent', + index_together={('inventory_update', 'job_created', 'counter'), ('inventory_update', 'job_created', 'uuid')}, + ), + migrations.AlterIndexTogether( + name='jobevent', + index_together={ + ('job', 'job_created', 'counter'), + ('job', 'job_created', 'uuid'), + ('job', 'job_created', 'event'), + ('job', 'job_created', 'parent_uuid'), + }, + ), + migrations.AlterIndexTogether( + name='projectupdateevent', + index_together={ + ('project_update', 'job_created', 'uuid'), + ('project_update', 'job_created', 'event'), + ('project_update', 'job_created', 'counter'), + }, + ), + migrations.AlterIndexTogether( + name='systemjobevent', + index_together={('system_job', 'job_created', 'uuid'), ('system_job', 'job_created', 'counter')}, + ), ] diff --git a/awx/main/models/events.py b/awx/main/models/events.py index de93d2ea3a..4cf78ebc0c 100644 --- a/awx/main/models/events.py +++ b/awx/main/models/events.py @@ -272,6 +272,10 @@ class BasePlaybookEvent(CreatedModifiedModel): null=True, default=None, editable=False, + ) + modified = models.DateTimeField( + default=None, + editable=False, db_index=True, ) @@ -366,14 +370,24 @@ class BasePlaybookEvent(CreatedModifiedModel): # find parent links and progagate changed=T and failed=T changed = ( - job.job_events.filter(changed=True).exclude(parent_uuid=None).only('parent_uuid').values_list('parent_uuid', flat=True).distinct() + job.get_event_queryset() + .filter(changed=True) + .exclude(parent_uuid=None) + .only('parent_uuid') + .values_list('parent_uuid', flat=True) + .distinct() ) # noqa failed = ( - job.job_events.filter(failed=True).exclude(parent_uuid=None).only('parent_uuid').values_list('parent_uuid', flat=True).distinct() + job.get_event_queryset() + .filter(failed=True) + .exclude(parent_uuid=None) + .only('parent_uuid') + .values_list('parent_uuid', flat=True) + .distinct() ) # noqa - JobEvent.objects.filter(job_id=self.job_id, uuid__in=changed).update(changed=True) - JobEvent.objects.filter(job_id=self.job_id, uuid__in=failed).update(failed=True) + job.get_event_queryset().filter(uuid__in=changed).update(changed=True) + job.get_event_queryset().filter(uuid__in=failed).update(failed=True) # send success/failure notifications when we've finished handling the playbook_on_stats event from awx.main.tasks import handle_success_and_failure_notifications # circular import @@ -468,11 +482,10 @@ class JobEvent(BasePlaybookEvent): app_label = 'main' ordering = ('pk',) index_together = [ - ('job', 'event'), - ('job', 'uuid'), - ('job', 'start_line'), - ('job', 'end_line'), - ('job', 'parent_uuid'), + ('job', 'job_created', 'event'), + ('job', 'job_created', 'uuid'), + ('job', 'job_created', 'parent_uuid'), + ('job', 'job_created', 'counter'), ] id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') @@ -482,6 +495,7 @@ class JobEvent(BasePlaybookEvent): null=True, on_delete=models.DO_NOTHING, editable=False, + db_index=False, ) host = models.ForeignKey( 'Host', @@ -599,18 +613,18 @@ class ProjectUpdateEvent(BasePlaybookEvent): app_label = 'main' ordering = ('pk',) index_together = [ - ('project_update', 'event'), - ('project_update', 'uuid'), - ('project_update', 'start_line'), - ('project_update', 'end_line'), + ('project_update', 'job_created', 'event'), + ('project_update', 'job_created', 'uuid'), + ('project_update', 'job_created', 'counter'), ] id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') project_update = models.ForeignKey( 'ProjectUpdate', related_name='project_update_events', - on_delete=models.CASCADE, + on_delete=models.DO_NOTHING, editable=False, + db_index=False, ) job_created = models.DateTimeField(null=True, editable=False) @@ -666,6 +680,16 @@ class BaseCommandEvent(CreatedModifiedModel): default=0, editable=False, ) + created = models.DateTimeField( + null=True, + default=None, + editable=False, + ) + modified = models.DateTimeField( + default=None, + editable=False, + db_index=True, + ) def __str__(self): return u'%s @ %s' % (self.get_event_display(), self.created.isoformat()) @@ -728,10 +752,9 @@ class AdHocCommandEvent(BaseCommandEvent): app_label = 'main' ordering = ('-pk',) index_together = [ - ('ad_hoc_command', 'event'), - ('ad_hoc_command', 'uuid'), - ('ad_hoc_command', 'start_line'), - ('ad_hoc_command', 'end_line'), + ('ad_hoc_command', 'job_created', 'event'), + ('ad_hoc_command', 'job_created', 'uuid'), + ('ad_hoc_command', 'job_created', 'counter'), ] EVENT_TYPES = [ @@ -778,8 +801,9 @@ class AdHocCommandEvent(BaseCommandEvent): ad_hoc_command = models.ForeignKey( 'AdHocCommand', related_name='ad_hoc_command_events', - on_delete=models.CASCADE, + on_delete=models.DO_NOTHING, editable=False, + db_index=False, ) host = models.ForeignKey( 'Host', @@ -828,17 +852,17 @@ class InventoryUpdateEvent(BaseCommandEvent): app_label = 'main' ordering = ('-pk',) index_together = [ - ('inventory_update', 'uuid'), - ('inventory_update', 'start_line'), - ('inventory_update', 'end_line'), + ('inventory_update', 'job_created', 'uuid'), + ('inventory_update', 'job_created', 'counter'), ] id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') inventory_update = models.ForeignKey( 'InventoryUpdate', related_name='inventory_update_events', - on_delete=models.CASCADE, + on_delete=models.DO_NOTHING, editable=False, + db_index=False, ) job_created = models.DateTimeField(null=True, editable=False) @@ -873,17 +897,17 @@ class SystemJobEvent(BaseCommandEvent): app_label = 'main' ordering = ('-pk',) index_together = [ - ('system_job', 'uuid'), - ('system_job', 'start_line'), - ('system_job', 'end_line'), + ('system_job', 'job_created', 'uuid'), + ('system_job', 'job_created', 'counter'), ] id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID') system_job = models.ForeignKey( 'SystemJob', related_name='system_job_events', - on_delete=models.CASCADE, + on_delete=models.DO_NOTHING, editable=False, + db_index=False, ) job_created = models.DateTimeField(null=True, editable=False)