From 38bf174bda042666b5fb719b00964be7a1f75cf7 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 28 Nov 2018 17:56:04 -0500 Subject: [PATCH] don't reap jobs that aren't running this is a simple sanity check, but it should help us avoid shooting ourselves in the foot in complicated scenarios, such as: 1. A dispatcher worker is running a job, and it's killed with `kill -9` 2. The dispatcher attempts to reap jobs with a matching celery_task_id 3. The associated sync project update has the *same* celery_task_id (an implementation detail of how we implemented that), and it ends up getting reaped _even though_ it's already finished and has status=successful --- awx/main/dispatch/reaper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/awx/main/dispatch/reaper.py b/awx/main/dispatch/reaper.py index f37a62290f..39991b6501 100644 --- a/awx/main/dispatch/reaper.py +++ b/awx/main/dispatch/reaper.py @@ -11,6 +11,9 @@ logger = logging.getLogger('awx.main.dispatch') def reap_job(j, status): + if UnifiedJob.objects.get(id=j.id).status not in ('running', 'waiting'): + # just in case, don't reap jobs that aren't running + return j.status = status j.start_args = '' # blank field to remove encrypted passwords j.job_explanation += ' '.join((