External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-16 12:20:45 -03:30
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #547 from ryanpetrello/fix-cleanup-memory-usage

Browse Source 
[3.2.2] Backport (from awx) various memory optimizations for job cleanup
This commit is contained in:
Ryan Petrello 2017-10-26 10:39:58 -04:00 committed by GitHub
commit 9e8ac3b09b
1 changed files with 29 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
awx/main/management/commands/cleanup_jobs.py
View File

@ -65,7 +65,8 @@ class Command(NoArgsCommand):
#jobs_qs = Job.objects.exclude(status__in=('pending', 'running'))
#jobs_qs = jobs_qs.filter(created__lte=self.cutoff)
skipped, deleted = 0, 0
for job in Job.objects.all():
jobs = Job.objects.filter(created__lt=self.cutoff)
for job in jobs.iterator():
job_display = '"%s" (%d host summaries, %d events)' % \
(unicode(job),
job.job_host_summaries.count(), job.job_events.count())
@ -73,21 +74,20 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s job %s', action_text, job.status, job_display)
skipped += 1
elif job.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, job_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, job_display)
if not self.dry_run:
job.delete()
deleted += 1
skipped += Job.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_ad_hoc_commands(self):
skipped, deleted = 0, 0
for ad_hoc_command in AdHocCommand.objects.all():
ad_hoc_commands = AdHocCommand.objects.filter(created__lt=self.cutoff)
for ad_hoc_command in ad_hoc_commands.iterator():
ad_hoc_command_display = '"%s" (%d events)' % \
(unicode(ad_hoc_command),
ad_hoc_command.ad_hoc_command_events.count())
@ -95,21 +95,20 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s ad hoc command %s', action_text, ad_hoc_command.status, ad_hoc_command_display)
skipped += 1
elif ad_hoc_command.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, ad_hoc_command_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, ad_hoc_command_display)
if not self.dry_run:
ad_hoc_command.delete()
deleted += 1
skipped += AdHocCommand.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_project_updates(self):
skipped, deleted = 0, 0
for pu in ProjectUpdate.objects.all():
project_updates = ProjectUpdate.objects.filter(created__lt=self.cutoff)
for pu in project_updates.iterator():
pu_display = '"%s" (type %s)' % (unicode(pu), unicode(pu.launch_type))
if pu.status in ('pending', 'waiting', 'running'):
action_text = 'would skip' if self.dry_run else 'skipping'
@ -119,21 +118,20 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, pu_display)
skipped += 1
elif pu.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, pu_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, pu_display)
if not self.dry_run:
pu.delete()
deleted += 1
skipped += ProjectUpdate.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_inventory_updates(self):
skipped, deleted = 0, 0
for iu in InventoryUpdate.objects.all():
inventory_updates = InventoryUpdate.objects.filter(created__lt=self.cutoff)
for iu in inventory_updates.iterator():
iu_display = '"%s" (source %s)' % (unicode(iu), unicode(iu.source))
if iu.status in ('pending', 'waiting', 'running'):
action_text = 'would skip' if self.dry_run else 'skipping'
@ -143,36 +141,33 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, iu_display)
skipped += 1
elif iu.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, iu_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, iu_display)
if not self.dry_run:
iu.delete()
deleted += 1
skipped += InventoryUpdate.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_management_jobs(self):
skipped, deleted = 0, 0
for sj in SystemJob.objects.all():
system_jobs = SystemJob.objects.filter(created__lt=self.cutoff)
for sj in system_jobs.iterator():
sj_display = '"%s" (type %s)' % (unicode(sj), unicode(sj.job_type))
if sj.status in ('pending', 'waiting', 'running'):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s system_job %s', action_text, sj.status, sj_display)
skipped += 1
elif sj.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, sj_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, sj_display)
if not self.dry_run:
sj.delete()
deleted += 1
skipped += SystemJob.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def init_logging(self):
@ -187,7 +182,8 @@ class Command(NoArgsCommand):
def cleanup_workflow_jobs(self):
skipped, deleted = 0, 0
for workflow_job in WorkflowJob.objects.all():
workflow_jobs = WorkflowJob.objects.filter(created__lt=self.cutoff)
for workflow_job in workflow_jobs.iterator():
workflow_job_display = '"{}" ({} nodes)'.format(
unicode(workflow_job),
workflow_job.workflow_nodes.count())
@ -195,21 +191,20 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s job %s', action_text, workflow_job.status, workflow_job_display)
skipped += 1
elif workflow_job.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, workflow_job_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, workflow_job_display)
if not self.dry_run:
workflow_job.delete()
deleted += 1
skipped += WorkflowJob.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
def cleanup_notifications(self):
skipped, deleted = 0, 0
for notification in Notification.objects.all():
notifications = Notification.objects.filter(created__lt=self.cutoff)
for notification in notifications.iterator():
notification_display = '"{}" (started {}, {} type, {} sent)'.format(
unicode(notification), unicode(notification.created),
notification.notification_type, notification.notifications_sent)
@ -217,16 +212,14 @@ class Command(NoArgsCommand):
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s notification %s', action_text, notification.status, notification_display)
skipped += 1
elif notification.created >= self.cutoff:
action_text = 'would skip' if self.dry_run else 'skipping'
self.logger.debug('%s %s', action_text, notification_display)
skipped += 1
else:
action_text = 'would delete' if self.dry_run else 'deleting'
self.logger.info('%s %s', action_text, notification_display)
if not self.dry_run:
notification.delete()
deleted += 1
skipped += Notification.objects.filter(created__gte=self.cutoff).count()
return skipped, deleted
@transaction.atomic
@ -255,3 +248,4 @@ class Command(NoArgsCommand):
self.logger.log(99, '%s: %d would be deleted, %d would be skipped.', m.replace('_', ' '), deleted, skipped)
else:
self.logger.log(99, '%s: %d deleted, %d skipped.', m.replace('_', ' '), deleted, skipped)