just like we fail running tasks fail waiting tasks

* Associate the celery_id with the job at the earliest point possible.
This ensures that a waiting job has a celery id. Thus, we are free to
fail waiting jobs that don't have a celery id.

awx/main/scheduler/__init__.py

@@ -60,8 +60,7 @@ class TaskManager():
     '''
     Tasks that are running and SHOULD have a celery task.
     '''
-    def get_running_tasks(self):
-        status_list = ('running',)
+    def get_running_tasks(self, status_list=('running',)):
 
         jobs = JobDict.filter_partial(status=status_list)
         inventory_updates = InventoryUpdateDict.filter_partial(status=status_list)
@@ -216,15 +215,15 @@ class TaskManager():
         else:
             if type(job_obj) is WorkflowJob:
                 job_obj.status = 'running'
-
-            job_obj.save()
+            else:
+                celery_task_id = job_obj.start_celery_task(opts, error_callback=error_handler, success_callback=success_handler)
+                job_obj.celery_task_id = celery_task_id
 
             self.consume_capacity(task)
+            job_obj.save()
 
         def post_commit():
             job_obj.websocket_emit_status(job_obj.status)
-            if job_obj.status != 'failed':
-                job_obj.start_celery_task(opts, error_callback=error_handler, success_callback=success_handler)
 
         connection.on_commit(post_commit)
 

awx/main/scheduler/tasks.py

@@ -52,7 +52,7 @@ def run_fail_inconsistent_running_jobs():
                 # TODO: Failed to contact celery. We should surface this.
                 return None
 
-            all_running_sorted_tasks = scheduler.get_running_tasks()
+            all_running_sorted_tasks = scheduler.get_running_tasks(status_list=('running', 'waiting',))
             scheduler.process_celery_tasks(active_tasks, all_running_sorted_tasks)
         except DatabaseError:
             return