periodically run orphaned task cleanup as part of the scheduler

Running orphaned task cleanup within its own scheduled task via
celery-beat causes a race-y lock contention between the cleanup task and
the task scheduler.  Unfortunately, the scheduler and the cleanup task
both run at similar intervals, so this race condition is fairly easy to
hit.  At best, it results in situations where the scheduler is
regularly delayed 20s; depending on timing, this can cause situations
where task execution is needlessly delayed a minute+.  At worst, it can
result in situations where the scheduler is never able to schedule
tasks.

This change implements the cleanup as a periodic block of code in the
scheduler itself that tracks its "last run" time in memcached (by
default, it performs a cleanup every 60 seconds)

see: #6534

awx/main/scheduler/tasks.py

@@ -1,19 +1,12 @@
 
 # Python
 import logging
-import json
-
-# Django
-from django.db import transaction
-from django.utils.timezone import now as tz_now
-from awx.main.utils.pglock import advisory_lock
 
 # Celery
 from celery import task
 
 # AWX
-from awx.main.scheduler import TaskManager 
-from django.core.cache import cache
+from awx.main.scheduler import TaskManager
 
 logger = logging.getLogger('awx.main.scheduler')
 
@@ -36,24 +29,3 @@ def run_job_complete(job_id):
 def run_task_manager():
     logger.debug("Running Tower task manager.")
     TaskManager().schedule()
-
-
-@task
-def run_fail_inconsistent_running_jobs():
-    logger.debug("Running task to fail inconsistent running jobs.")
-    with transaction.atomic():
-        # Lock
-        with advisory_lock('task_manager_lock', wait=False) as acquired:
-            if acquired is False:
-                return
-
-            scheduler = TaskManager()
-            celery_task_start_time = tz_now()
-            active_task_queues, active_tasks = scheduler.get_active_tasks()
-            cache.set("active_celery_tasks", json.dumps(active_task_queues))
-            if active_tasks is None:
-                # TODO: Failed to contact celery. We should surface this.
-                return None
-
-            all_running_sorted_tasks = scheduler.get_running_tasks()
-            scheduler.process_celery_tasks(celery_task_start_time, active_tasks, all_running_sorted_tasks)