diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py index a0008b9504..595abcc9e7 100644 --- a/awx/main/management/commands/run_dispatcher.py +++ b/awx/main/management/commands/run_dispatcher.py @@ -69,13 +69,29 @@ class Command(BaseCommand): return TaskResult() + sched_file = '/var/lib/awx/beat.db' app = Celery() app.conf.BROKER_URL = settings.BROKER_URL app.conf.CELERY_TASK_RESULT_EXPIRES = False + + # celery in py3 seems to have a bug where the celerybeat schedule + # shelve can become corrupted; we've _only_ seen this in Ubuntu and py36 + # it can be avoided by detecting and removing the corrupted file + # at some point, we'll just stop using celerybeat, because it's clearly + # buggy, too -_- + # + # https://github.com/celery/celery/issues/4777 + sched = AWXScheduler(schedule_filename=sched_file, app=app) + try: + sched.setup_schedule() + except Exception: + logger.exception('{} is corrupted, removing.'.format(sched_file)) + sched._remove_db() + beat.Beat( 30, app, - schedule='/var/lib/awx/beat.db', scheduler_cls=AWXScheduler + schedule=sched_file, scheduler_cls=AWXScheduler ).run() def handle(self, *arg, **options):