diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py index 970446a0e5..b57034b9f8 100644 --- a/awx/main/management/commands/run_dispatcher.py +++ b/awx/main/management/commands/run_dispatcher.py @@ -10,6 +10,7 @@ from django.core.management.base import BaseCommand from django.db import connection as django_connection, connections from kombu import Exchange, Queue +from awx.main.utils.handlers import AWXProxyHandler from awx.main.dispatch import get_local_queuename, reaper from awx.main.dispatch.control import Control from awx.main.dispatch.kombu import Connection @@ -121,6 +122,12 @@ class Command(BaseCommand): reaper.reap() consumer = None + + # don't ship external logs inside the dispatcher's parent process + # this exists to work around a race condition + deadlock bug on fork + # in cpython itself: + # https://bugs.python.org/issue37429 + AWXProxyHandler.disable() with Connection(settings.BROKER_URL) as conn: try: bcast = 'tower_broadcast_all' diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index aac46966b5..dae0de8e79 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -6,6 +6,7 @@ import logging import json import requests import time +import threading import socket import select from urllib import parse as urlparse @@ -286,6 +287,8 @@ class AWXProxyHandler(logging.Handler): Parameters match same parameters in the actualized handler classes. ''' + thread_local = threading.local() + def __init__(self, **kwargs): # TODO: process 'level' kwarg super(AWXProxyHandler, self).__init__(**kwargs) @@ -322,8 +325,9 @@ class AWXProxyHandler(logging.Handler): return self._handler def emit(self, record): - actual_handler = self.get_handler() - return actual_handler.emit(record) + if AWXProxyHandler.thread_local.enabled: + actual_handler = self.get_handler() + return actual_handler.emit(record) def perform_test(self, custom_settings): """ @@ -353,6 +357,13 @@ class AWXProxyHandler(logging.Handler): except RequestException as e: raise LoggingConnectivityException(str(e)) + @classmethod + def disable(cls): + cls.thread_local.enabled = False + + +AWXProxyHandler.thread_local.enabled = True + ColorHandler = logging.StreamHandler