External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-11 10:00:01 -03:30
Code Issues Packages Projects Releases Wiki Activity

add/remove indexes, more get_event_querset()

Browse Source 
* Do not cascade delete unified job events. We will clean those up in
cleanup_job runs
* Add limit pagination to all unified job events endpoints
This commit is contained in:
Chris Meyers 2021-05-20 21:07:41 -04:00 committed by Jim Ladd
parent c429563126
commit 2131703ca0
6 changed files with 190 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
awx/api/pagination.py
View File

@ -103,7 +103,7 @@ class LimitPagination(pagination.BasePagination):
return self.default_limit
class JobEventPagination(Pagination):
class UnifiedJobEventPagination(Pagination):
"""
By default, use Pagination for all operations.
If `limit` query parameter specified use LimitPagination

8
awx/api/serializers.py
View File

@ -3044,7 +3044,7 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer):
res = super(JobSerializer, self).get_related(obj)
res.update(
dict(
job_events=self.reverse('api:job_job_events_list', kwargs={'pk': obj.pk}),
job_events=self.reverse('api:job_job_events_list', kwargs={'pk': obj.pk}), # TODO: consider adding job_created
job_host_summaries=self.reverse('api:job_job_host_summaries_list', kwargs={'pk': obj.pk}),
activity_stream=self.reverse('api:job_activity_stream_list', kwargs={'pk': obj.pk}),
notifications=self.reverse('api:job_notifications_list', kwargs={'pk': obj.pk}),
@ -3111,8 +3111,8 @@ class JobDetailSerializer(JobSerializer):
fields = ('*', 'host_status_counts', 'playbook_counts', 'custom_virtualenv')
def get_playbook_counts(self, obj):
task_count = obj.job_events.filter(event='playbook_on_task_start').count()
play_count = obj.job_events.filter(event='playbook_on_play_start').count()
task_count = obj.get_event_queryset().filter(event='playbook_on_task_start').count()
play_count = obj.get_event_queryset().filter(event='playbook_on_play_start').count()
data = {'play_count': play_count, 'task_count': task_count}
@ -3120,7 +3120,7 @@ class JobDetailSerializer(JobSerializer):
def get_host_status_counts(self, obj):
try:
counts = obj.job_events.only('event_data').get(event='playbook_on_stats').get_host_status_counts()
counts = obj.get_event_queryset().only('event_data').get(event='playbook_on_stats').get_host_status_counts()
except JobEvent.DoesNotExist:
counts = {}

9
awx/api/views/__init__.py
View File

@ -172,7 +172,7 @@ from awx.api.views.root import ( # noqa
ApiV2AttachView,
)
from awx.api.views.webhooks import WebhookKeyView, GithubWebhookReceiver, GitlabWebhookReceiver # noqa
from awx.api.pagination import JobEventPagination
from awx.api.pagination import UnifiedJobEventPagination
logger = logging.getLogger('awx.api.views')
@ -888,6 +888,7 @@ class ProjectUpdateEventsList(SubListAPIView):
relationship = 'project_update_events'
name = _('Project Update Events List')
search_fields = ('stdout',)
pagination_class = UnifiedJobEventPagination
def finalize_response(self, request, response, *args, **kwargs):
response['X-UI-Max-Events'] = settings.MAX_UI_JOB_EVENTS
@ -907,6 +908,7 @@ class SystemJobEventsList(SubListAPIView):
relationship = 'system_job_events'
name = _('System Job Events List')
search_fields = ('stdout',)
pagination_class = UnifiedJobEventPagination
def finalize_response(self, request, response, *args, **kwargs):
response['X-UI-Max-Events'] = settings.MAX_UI_JOB_EVENTS
@ -3622,7 +3624,7 @@ class JobRelaunch(RetrieveAPIView):
status=status.HTTP_400_BAD_REQUEST,
)
host_qs = obj.retry_qs(retry_hosts)
if not obj.job_events.filter(event='playbook_on_stats').exists():
if not obj.get_event_queryset().filter(event='playbook_on_stats').exists():
return Response(
{'hosts': _('Cannot retry on {status_value} hosts, playbook stats not available.').format(status_value=retry_hosts)},
status=status.HTTP_400_BAD_REQUEST,
@ -3833,7 +3835,7 @@ class GroupJobEventsList(BaseJobEventsList):
class JobJobEventsList(BaseJobEventsList):
parent_model = models.Job
pagination_class = JobEventPagination
pagination_class = UnifiedJobEventPagination
def get_queryset(self):
job = self.get_parent_object()
@ -4021,6 +4023,7 @@ class BaseAdHocCommandEventsList(NoTruncateMixin, SubListAPIView):
relationship = 'ad_hoc_command_events'
name = _('Ad Hoc Command Events List')
search_fields = ('stdout',)
pagination_class = UnifiedJobEventPagination
def get_queryset(self):
parent = self.get_parent_object()

4
awx/api/views/inventory.py
View File

@ -38,6 +38,9 @@ from awx.api.serializers import (
)
from awx.api.views.mixin import RelatedJobsPreventDeleteMixin, ControlledByScmMixin
from awx.api.pagination import UnifiedJobEventPagination
logger = logging.getLogger('awx.api.views.organization')
@ -49,6 +52,7 @@ class InventoryUpdateEventsList(SubListAPIView):
relationship = 'inventory_update_events'
name = _('Inventory Update Events List')
search_fields = ('stdout',)
pagination_class = UnifiedJobEventPagination
def get_queryset(self):
iu = self.get_parent_object()

132
awx/main/migrations/0144_event_partitions.py
View File

@ -47,12 +47,17 @@ def migrate_event_data(apps, schema_editor):
cursor.execute(f'DROP TABLE tmp_{tblname}')
# let's go ahead and add and subtract a few indexes while we're here
cursor.execute(f'CREATE INDEX {tblname}_modified_idx ON {tblname} (modified);')
# recreate primary key constraint
cursor.execute(f'ALTER TABLE ONLY {tblname} ' f'ADD CONSTRAINT {tblname}_pkey_new PRIMARY KEY (id, job_created);')
with connection.cursor() as cursor:
"""
Big int migration introduced the brin index main_jobevent_job_id_brin_idx index. For upgardes, we drop the index, new installs do nothing.
I have seen the second index in my dev environment. I can not find where in the code it was created. Drop it just in case
"""
cursor.execute('DROP INDEX IF EXISTS main_jobevent_job_id_brin_idx')
cursor.execute('DROP INDEX IF EXISTS main_jobevent_job_id_idx')
class FakeAddField(migrations.AddField):
def database_forwards(self, *args):
@ -94,11 +99,6 @@ class Migration(migrations.Migration):
name='job_created',
field=models.DateTimeField(null=True, editable=False),
),
migrations.AlterField(
model_name='jobevent',
name='job',
field=models.ForeignKey(editable=False, null=True, on_delete=models.deletion.DO_NOTHING, related_name='job_events', to='main.Job'),
),
migrations.CreateModel(
name='UnpartitionedAdHocCommandEvent',
fields=[],
@ -149,4 +149,120 @@ class Migration(migrations.Migration):
},
bases=('main.systemjobevent',),
),
migrations.AlterField(
model_name='adhoccommandevent',
name='ad_hoc_command',
field=models.ForeignKey(
db_index=False, editable=False, on_delete=models.deletion.DO_NOTHING, related_name='ad_hoc_command_events', to='main.AdHocCommand'
),
),
migrations.AlterField(
model_name='adhoccommandevent',
name='created',
field=models.DateTimeField(default=None, editable=False, null=True),
),
migrations.AlterField(
model_name='adhoccommandevent',
name='modified',
field=models.DateTimeField(db_index=True, default=None, editable=False),
),
migrations.AlterField(
model_name='inventoryupdateevent',
name='created',
field=models.DateTimeField(default=None, editable=False, null=True),
),
migrations.AlterField(
model_name='inventoryupdateevent',
name='inventory_update',
field=models.ForeignKey(
db_index=False, editable=False, on_delete=models.deletion.DO_NOTHING, related_name='inventory_update_events', to='main.InventoryUpdate'
),
),
migrations.AlterField(
model_name='inventoryupdateevent',
name='modified',
field=models.DateTimeField(db_index=True, default=None, editable=False),
),
migrations.AlterField(
model_name='jobevent',
name='created',
field=models.DateTimeField(default=None, editable=False, null=True),
),
migrations.AlterField(
model_name='jobevent',
name='job',
field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=models.deletion.DO_NOTHING, related_name='job_events', to='main.Job'),
),
migrations.AlterField(
model_name='jobevent',
name='modified',
field=models.DateTimeField(db_index=True, default=None, editable=False),
),
migrations.AlterField(
model_name='projectupdateevent',
name='created',
field=models.DateTimeField(default=None, editable=False, null=True),
),
migrations.AlterField(
model_name='projectupdateevent',
name='modified',
field=models.DateTimeField(db_index=True, default=None, editable=False),
),
migrations.AlterField(
model_name='projectupdateevent',
name='project_update',
field=models.ForeignKey(
db_index=False, editable=False, on_delete=models.deletion.DO_NOTHING, related_name='project_update_events', to='main.ProjectUpdate'
),
),
migrations.AlterField(
model_name='systemjobevent',
name='created',
field=models.DateTimeField(default=None, editable=False, null=True),
),
migrations.AlterField(
model_name='systemjobevent',
name='modified',
field=models.DateTimeField(db_index=True, default=None, editable=False),
),
migrations.AlterField(
model_name='systemjobevent',
name='system_job',
field=models.ForeignKey(
db_index=False, editable=False, on_delete=models.deletion.DO_NOTHING, related_name='system_job_events', to='main.SystemJob'
),
),
migrations.AlterIndexTogether(
name='adhoccommandevent',
index_together={
('ad_hoc_command', 'job_created', 'event'),
('ad_hoc_command', 'job_created', 'counter'),
('ad_hoc_command', 'job_created', 'uuid'),
},
),
migrations.AlterIndexTogether(
name='inventoryupdateevent',
index_together={('inventory_update', 'job_created', 'counter'), ('inventory_update', 'job_created', 'uuid')},
),
migrations.AlterIndexTogether(
name='jobevent',
index_together={
('job', 'job_created', 'counter'),
('job', 'job_created', 'uuid'),
('job', 'job_created', 'event'),
('job', 'job_created', 'parent_uuid'),
},
),
migrations.AlterIndexTogether(
name='projectupdateevent',
index_together={
('project_update', 'job_created', 'uuid'),
('project_update', 'job_created', 'event'),
('project_update', 'job_created', 'counter'),
},
),
migrations.AlterIndexTogether(
name='systemjobevent',
index_together={('system_job', 'job_created', 'uuid'), ('system_job', 'job_created', 'counter')},
),
]

78
awx/main/models/events.py
View File

@ -272,6 +272,10 @@ class BasePlaybookEvent(CreatedModifiedModel):
null=True,
default=None,
editable=False,
)
modified = models.DateTimeField(
default=None,
editable=False,
db_index=True,
)
@ -366,14 +370,24 @@ class BasePlaybookEvent(CreatedModifiedModel):
# find parent links and progagate changed=T and failed=T
changed = (
job.job_events.filter(changed=True).exclude(parent_uuid=None).only('parent_uuid').values_list('parent_uuid', flat=True).distinct()
job.get_event_queryset()
.filter(changed=True)
.exclude(parent_uuid=None)
.only('parent_uuid')
.values_list('parent_uuid', flat=True)
.distinct()
) # noqa
failed = (
job.job_events.filter(failed=True).exclude(parent_uuid=None).only('parent_uuid').values_list('parent_uuid', flat=True).distinct()
job.get_event_queryset()
.filter(failed=True)
.exclude(parent_uuid=None)
.only('parent_uuid')
.values_list('parent_uuid', flat=True)
.distinct()
) # noqa
JobEvent.objects.filter(job_id=self.job_id, uuid__in=changed).update(changed=True)
JobEvent.objects.filter(job_id=self.job_id, uuid__in=failed).update(failed=True)
job.get_event_queryset().filter(uuid__in=changed).update(changed=True)
job.get_event_queryset().filter(uuid__in=failed).update(failed=True)
# send success/failure notifications when we've finished handling the playbook_on_stats event
from awx.main.tasks import handle_success_and_failure_notifications # circular import
@ -468,11 +482,10 @@ class JobEvent(BasePlaybookEvent):
app_label = 'main'
ordering = ('pk',)
index_together = [
('job', 'event'),
('job', 'uuid'),
('job', 'start_line'),
('job', 'end_line'),
('job', 'parent_uuid'),
('job', 'job_created', 'event'),
('job', 'job_created', 'uuid'),
('job', 'job_created', 'parent_uuid'),
('job', 'job_created', 'counter'),
]
id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')
@ -482,6 +495,7 @@ class JobEvent(BasePlaybookEvent):
null=True,
on_delete=models.DO_NOTHING,
editable=False,
db_index=False,
)
host = models.ForeignKey(
'Host',
@ -599,18 +613,18 @@ class ProjectUpdateEvent(BasePlaybookEvent):
app_label = 'main'
ordering = ('pk',)
index_together = [
('project_update', 'event'),
('project_update', 'uuid'),
('project_update', 'start_line'),
('project_update', 'end_line'),
('project_update', 'job_created', 'event'),
('project_update', 'job_created', 'uuid'),
('project_update', 'job_created', 'counter'),
]
id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')
project_update = models.ForeignKey(
'ProjectUpdate',
related_name='project_update_events',
on_delete=models.CASCADE,
on_delete=models.DO_NOTHING,
editable=False,
db_index=False,
)
job_created = models.DateTimeField(null=True, editable=False)
@ -666,6 +680,16 @@ class BaseCommandEvent(CreatedModifiedModel):
default=0,
editable=False,
)
created = models.DateTimeField(
null=True,
default=None,
editable=False,
)
modified = models.DateTimeField(
default=None,
editable=False,
db_index=True,
)
def __str__(self):
return u'%s @ %s' % (self.get_event_display(), self.created.isoformat())
@ -728,10 +752,9 @@ class AdHocCommandEvent(BaseCommandEvent):
app_label = 'main'
ordering = ('-pk',)
index_together = [
('ad_hoc_command', 'event'),
('ad_hoc_command', 'uuid'),
('ad_hoc_command', 'start_line'),
('ad_hoc_command', 'end_line'),
('ad_hoc_command', 'job_created', 'event'),
('ad_hoc_command', 'job_created', 'uuid'),
('ad_hoc_command', 'job_created', 'counter'),
]
EVENT_TYPES = [
@ -778,8 +801,9 @@ class AdHocCommandEvent(BaseCommandEvent):
ad_hoc_command = models.ForeignKey(
'AdHocCommand',
related_name='ad_hoc_command_events',
on_delete=models.CASCADE,
on_delete=models.DO_NOTHING,
editable=False,
db_index=False,
)
host = models.ForeignKey(
'Host',
@ -828,17 +852,17 @@ class InventoryUpdateEvent(BaseCommandEvent):
app_label = 'main'
ordering = ('-pk',)
index_together = [
('inventory_update', 'uuid'),
('inventory_update', 'start_line'),
('inventory_update', 'end_line'),
('inventory_update', 'job_created', 'uuid'),
('inventory_update', 'job_created', 'counter'),
]
id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')
inventory_update = models.ForeignKey(
'InventoryUpdate',
related_name='inventory_update_events',
on_delete=models.CASCADE,
on_delete=models.DO_NOTHING,
editable=False,
db_index=False,
)
job_created = models.DateTimeField(null=True, editable=False)
@ -873,17 +897,17 @@ class SystemJobEvent(BaseCommandEvent):
app_label = 'main'
ordering = ('-pk',)
index_together = [
('system_job', 'uuid'),
('system_job', 'start_line'),
('system_job', 'end_line'),
('system_job', 'job_created', 'uuid'),
('system_job', 'job_created', 'counter'),
]
id = models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')
system_job = models.ForeignKey(
'SystemJob',
related_name='system_job_events',
on_delete=models.CASCADE,
on_delete=models.DO_NOTHING,
editable=False,
db_index=False,
)
job_created = models.DateTimeField(null=True, editable=False)