External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-13 11:00:03 -03:30
Code Issues Packages Projects Releases Wiki Activity

Backport (from awx) various memory optimizations for job cleanup

Browse Source 
see: https://github.com/ansible/ansible-tower/issues/7751

0388568 Reduces the job to only looking at objects older than the cutoff date
0234311 missing colon and missing variable usage
399e0e5 switching to iterator and adding the missed Job cleanup
0cd34c1 jobs take count of gte cutoff, process only lt cutoff
This commit is contained in:
Mike McMahon 2017-09-07 13:41:43 -07:00 committed by Ryan Petrello
parent 70919638ba
commit d3a7bec674
No known key found for this signature in database
GPG Key ID: F2AA5F2122351777
1 changed files with 29 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
awx/main/management/commands/cleanup_jobs.py
View File

@ -65,7 +65,8 @@ class Command(NoArgsCommand):
#jobs_qs = Job.objects.exclude(status__in=('pending', 'running'))
#jobs_qs = jobs_qs.filter(created__lte=self.cutoff)
skipped, deleted = 0, 0
for job in Job.objects.all():
jobs = Job.objects.filter(created__lt=self.cutoff)
for job in jobs.iterator():
job_display = '"%s" (%d host summaries, %d events)' % \
(unicode(job),
job.job_host_summaries.count(), job.job_events.count())
@ -73,21 +74,20 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s job %s', action_text, job.status, job_display)
skipped += 1
elif job.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, job_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, job_display)
if not self.dry_run:
job.delete()
deleted += 1
skipped += Job.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_ad_hoc_commands(self):
skipped, deleted = 0, 0
for ad_hoc_command in AdHocCommand.objects.all():
ad_hoc_commands = AdHocCommand.objects.filter(created__lt=self.cutoff)
for ad_hoc_command in ad_hoc_commands.iterator():
ad_hoc_command_display = '"%s" (%d events)' % \
(unicode(ad_hoc_command),
ad_hoc_command.ad_hoc_command_events.count())
@ -95,21 +95,20 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s ad hoc command %s', action_text, ad_hoc_command.status, ad_hoc_command_display)
skipped += 1
elif ad_hoc_command.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, ad_hoc_command_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, ad_hoc_command_display)
if not self.dry_run:
ad_hoc_command.delete()
deleted += 1
skipped += AdHocCommand.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_project_updates(self):
skipped, deleted = 0, 0
for pu in ProjectUpdate.objects.all():
project_updates = ProjectUpdate.objects.filter(created__lt=self.cutoff)
for pu in project_updates.iterator():
pu_display = '"%s" (type %s)' % (unicode(pu), unicode(pu.launch_type))
if pu.status in ('pending', 'waiting', 'running'):
action_text = 'would skip' if self.dry_run else 'skipping'
@ -119,21 +118,20 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, pu_display)
skipped += 1
elif pu.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, pu_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, pu_display)
if not self.dry_run:
pu.delete()
deleted += 1
skipped += ProjectUpdate.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_inventory_updates(self):
skipped, deleted = 0, 0
for iu in InventoryUpdate.objects.all():
inventory_updates = InventoryUpdate.objects.filter(created__lt=self.cutoff)
for iu in inventory_updates.iterator():
iu_display = '"%s" (source %s)' % (unicode(iu), unicode(iu.source))
if iu.status in ('pending', 'waiting', 'running'):
action_text = 'would skip' if self.dry_run else 'skipping'
@ -143,36 +141,33 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, iu_display)
skipped += 1
elif iu.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, iu_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, iu_display)
if not self.dry_run:
iu.delete()
deleted += 1
skipped += InventoryUpdate.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_management_jobs(self):
skipped, deleted = 0, 0
for sj in SystemJob.objects.all():
system_jobs = SystemJob.objects.filter(created__lt=self.cutoff)
for sj in system_jobs.iterator():
sj_display = '"%s" (type %s)' % (unicode(sj), unicode(sj.job_type))
if sj.status in ('pending', 'waiting', 'running'):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s system_job %s', action_text, sj.status, sj_display)
skipped += 1
elif sj.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, sj_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, sj_display)
if not self.dry_run:
sj.delete()
deleted += 1
skipped += SystemJob.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def init_logging(self):
@ -187,7 +182,8 @@ class Command(NoArgsCommand):
def cleanup_workflow_jobs(self):
skipped, deleted = 0, 0
for workflow_job in WorkflowJob.objects.all():
workflow_jobs = WorkflowJob.objects.filter(created__lt=self.cutoff)
for workflow_job in workflow_jobs.iterator():
workflow_job_display = '"{}" ({} nodes)'.format(
unicode(workflow_job),
workflow_job.workflow_nodes.count())
@ -195,21 +191,20 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s job %s', action_text, workflow_job.status, workflow_job_display)
skipped += 1
elif workflow_job.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, workflow_job_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, workflow_job_display)
if not self.dry_run:
workflow_job.delete()
deleted += 1
skipped += WorkflowJob.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_notifications(self):
skipped, deleted = 0, 0
for notification in Notification.objects.all():
notifications = Notification.objects.filter(created__lt=self.cutoff)
for notification in notifications.iterator():
notification_display = '"{}" (started {}, {} type, {} sent)'.format(
unicode(notification), unicode(notification.created),
notification.notification_type, notification.notifications_sent)
@ -217,16 +212,14 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s notification %s', action_text, notification.status, notification_display)
skipped += 1
elif notification.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, notification_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, notification_display)
if not self.dry_run:
notification.delete()
deleted += 1
skipped += Notification.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
@transaction.atomic
@ -255,3 +248,4 @@ class Command(NoArgsCommand):
self.logger.log(99, '%s: %d would be deleted, %d would be skipped.', m.replace('_', ' '), deleted, skipped)
else:
self.logger.log(99, '%s: %d deleted, %d skipped.', m.replace('_', ' '), deleted, skipped)