mirror of
https://github.com/ansible/awx.git
synced 2026-08-01 18:39:54 -02:30
Replace logging-related restart with dynamic handler
refactor existing handlers to be the related "real" handler classes, which are swapped out dynamically by external logger "proxy" handler class real handler swapout only done on setting change remove restart_local_services method get rid of uWSGI fifo file change TCP/UDP return type contract so that it mirrors the request futures object add details to socket error messages
This commit is contained in:
AlanCoding
@@ -13,40 +13,35 @@ import six
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
# loggly
|
||||
import traceback
|
||||
|
||||
# Django
|
||||
from django.conf import settings
|
||||
|
||||
# requests futures, a dependency used by these handlers
|
||||
from requests_futures.sessions import FuturesSession
|
||||
|
||||
# AWX
|
||||
from awx.main.utils.formatters import LogstashFormatter
|
||||
|
||||
|
||||
__all__ = ['HTTPSNullHandler', 'BaseHTTPSHandler', 'TCPHandler', 'UDPHandler',
|
||||
'configure_external_logger']
|
||||
__all__ = ['BaseHTTPSHandler', 'TCPHandler', 'UDPHandler',
|
||||
'AWXProxyHandler']
|
||||
|
||||
|
||||
logger = logging.getLogger('awx.main.utils.handlers')
|
||||
|
||||
# AWX external logging handler, generally designed to be used
|
||||
# with the accompanying LogstashHandler, derives from python-logstash library
|
||||
# Non-blocking request accomplished by FuturesSession, similar
|
||||
# to the loggly-python-handler library (not used)
|
||||
|
||||
# Translation of parameter names to names in Django settings
|
||||
# logging settings category, only those related to handler / log emission
|
||||
PARAM_NAMES = {
|
||||
'host': 'LOG_AGGREGATOR_HOST',
|
||||
'port': 'LOG_AGGREGATOR_PORT',
|
||||
'message_type': 'LOG_AGGREGATOR_TYPE',
|
||||
'username': 'LOG_AGGREGATOR_USERNAME',
|
||||
'password': 'LOG_AGGREGATOR_PASSWORD',
|
||||
'enabled_loggers': 'LOG_AGGREGATOR_LOGGERS',
|
||||
'indv_facts': 'LOG_AGGREGATOR_INDIVIDUAL_FACTS',
|
||||
'enabled_flag': 'LOG_AGGREGATOR_ENABLED',
|
||||
'tcp_timeout': 'LOG_AGGREGATOR_TCP_TIMEOUT',
|
||||
'verify_cert': 'LOG_AGGREGATOR_VERIFY_CERT',
|
||||
'lvl': 'LOG_AGGREGATOR_LEVEL',
|
||||
'protocol': 'LOG_AGGREGATOR_PROTOCOL'
|
||||
}
|
||||
|
||||
|
||||
@@ -58,13 +53,6 @@ class LoggingConnectivityException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class HTTPSNullHandler(logging.NullHandler):
|
||||
"Placeholder null handler to allow loading without database access"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
return super(HTTPSNullHandler, self).__init__()
|
||||
|
||||
|
||||
class VerboseThreadPoolExecutor(ThreadPoolExecutor):
|
||||
|
||||
last_log_emit = 0
|
||||
@@ -91,32 +79,25 @@ class VerboseThreadPoolExecutor(ThreadPoolExecutor):
|
||||
**kwargs)
|
||||
|
||||
|
||||
LEVEL_MAPPING = {
|
||||
'DEBUG': logging.DEBUG,
|
||||
'INFO': logging.INFO,
|
||||
'WARNING': logging.WARNING,
|
||||
'ERROR': logging.ERROR,
|
||||
'CRITICAL': logging.CRITICAL,
|
||||
}
|
||||
class SocketResult:
|
||||
'''
|
||||
A class to be the return type of methods that send data over a socket
|
||||
allows object to be used in the same way as a request futures object
|
||||
'''
|
||||
def __init__(self, ok, reason=None):
|
||||
self.ok = ok
|
||||
self.reason = reason
|
||||
|
||||
def result(self):
|
||||
return self
|
||||
|
||||
|
||||
class BaseHandler(logging.Handler):
|
||||
def __init__(self, **kwargs):
|
||||
def __init__(self, host=None, port=None, indv_facts=None, **kwargs):
|
||||
super(BaseHandler, self).__init__()
|
||||
for fd in PARAM_NAMES:
|
||||
setattr(self, fd, kwargs.get(fd, None))
|
||||
|
||||
@classmethod
|
||||
def from_django_settings(cls, settings, *args, **kwargs):
|
||||
for param, django_setting_name in PARAM_NAMES.items():
|
||||
kwargs[param] = getattr(settings, django_setting_name, None)
|
||||
return cls(*args, **kwargs)
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.indv_facts = indv_facts
|
||||
|
||||
def _send(self, payload):
|
||||
"""Actually send message to log aggregator.
|
||||
@@ -128,26 +109,11 @@ class BaseHandler(logging.Handler):
|
||||
return [self._send(json.loads(self.format(record)))]
|
||||
return [self._send(self.format(record))]
|
||||
|
||||
def _skip_log(self, logger_name):
|
||||
if self.host == '' or (not self.enabled_flag):
|
||||
return True
|
||||
# Don't send handler-related records.
|
||||
if logger_name == logger.name:
|
||||
return True
|
||||
# AWX log emission is only turned off by enablement setting
|
||||
if not logger_name.startswith('awx.analytics'):
|
||||
return False
|
||||
return self.enabled_loggers is None or logger_name[len('awx.analytics.'):] not in self.enabled_loggers
|
||||
|
||||
def emit(self, record):
|
||||
"""
|
||||
Emit a log record. Returns a list of zero or more
|
||||
implementation-specific objects for tests.
|
||||
"""
|
||||
if not record.name.startswith('awx.analytics') and record.levelno < LEVEL_MAPPING[self.lvl]:
|
||||
return []
|
||||
if self._skip_log(record.name):
|
||||
return []
|
||||
try:
|
||||
return self._format_and_send_record(record)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
@@ -181,6 +147,11 @@ class BaseHandler(logging.Handler):
|
||||
|
||||
|
||||
class BaseHTTPSHandler(BaseHandler):
|
||||
'''
|
||||
Originally derived from python-logstash library
|
||||
Non-blocking request accomplished by FuturesSession, similar
|
||||
to the loggly-python-handler library
|
||||
'''
|
||||
def _add_auth_information(self):
|
||||
if self.message_type == 'logstash':
|
||||
if not self.username:
|
||||
@@ -196,39 +167,20 @@ class BaseHTTPSHandler(BaseHandler):
|
||||
}
|
||||
self.session.headers.update(headers)
|
||||
|
||||
def __init__(self, fqdn=False, **kwargs):
|
||||
def __init__(self, fqdn=False, message_type=None, username=None, password=None,
|
||||
tcp_timeout=5, verify_cert=True, **kwargs):
|
||||
self.fqdn = fqdn
|
||||
self.message_type = message_type
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.tcp_timeout = tcp_timeout
|
||||
self.verify_cert = verify_cert
|
||||
super(BaseHTTPSHandler, self).__init__(**kwargs)
|
||||
self.session = FuturesSession(executor=VerboseThreadPoolExecutor(
|
||||
max_workers=2 # this is the default used by requests_futures
|
||||
))
|
||||
self._add_auth_information()
|
||||
|
||||
@classmethod
|
||||
def perform_test(cls, settings):
|
||||
"""
|
||||
Tests logging connectivity for the current logging settings.
|
||||
@raises LoggingConnectivityException
|
||||
"""
|
||||
handler = cls.from_django_settings(settings)
|
||||
handler.enabled_flag = True
|
||||
handler.setFormatter(LogstashFormatter(settings_module=settings))
|
||||
logger = logging.getLogger(__file__)
|
||||
fn, lno, func = logger.findCaller()
|
||||
record = logger.makeRecord('awx', 10, fn, lno,
|
||||
'AWX Connection Test', tuple(),
|
||||
None, func)
|
||||
futures = handler.emit(record)
|
||||
for future in futures:
|
||||
try:
|
||||
resp = future.result()
|
||||
if not resp.ok:
|
||||
raise LoggingConnectivityException(
|
||||
': '.join([str(resp.status_code), resp.reason or ''])
|
||||
)
|
||||
except RequestException as e:
|
||||
raise LoggingConnectivityException(str(e))
|
||||
|
||||
def _get_post_kwargs(self, payload_input):
|
||||
if self.message_type == 'splunk':
|
||||
# Splunk needs data nested under key "event"
|
||||
@@ -265,6 +217,10 @@ def _encode_payload_for_socket(payload):
|
||||
|
||||
|
||||
class TCPHandler(BaseHandler):
|
||||
def __init__(self, tcp_timeout=5, **kwargs):
|
||||
self.tcp_timeout = tcp_timeout
|
||||
super(TCPHandler, self).__init__(**kwargs)
|
||||
|
||||
def _send(self, payload):
|
||||
payload = _encode_payload_for_socket(payload)
|
||||
sok = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
@@ -273,39 +229,32 @@ class TCPHandler(BaseHandler):
|
||||
sok.setblocking(0)
|
||||
_, ready_to_send, _ = select.select([], [sok], [], float(self.tcp_timeout))
|
||||
if len(ready_to_send) == 0:
|
||||
logger.warning("Socket currently busy, failed to send message")
|
||||
sok.close()
|
||||
return
|
||||
sok.send(payload)
|
||||
ret = SocketResult(False, "Socket currently busy, failed to send message")
|
||||
logger.warning(ret.reason)
|
||||
else:
|
||||
sok.send(payload)
|
||||
ret = SocketResult(True) # success!
|
||||
except Exception as e:
|
||||
logger.exception("Error sending message from %s: %s" %
|
||||
(TCPHandler.__name__, e.message))
|
||||
sok.close()
|
||||
ret = SocketResult(False, "Error sending message from %s: %s" %
|
||||
(TCPHandler.__name__,
|
||||
' '.join(six.text_type(arg) for arg in e.args)))
|
||||
logger.exception(ret.reason)
|
||||
finally:
|
||||
sok.close()
|
||||
return ret
|
||||
|
||||
|
||||
class UDPHandler(BaseHandler):
|
||||
message = "Cannot determine if UDP messages are received."
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super(UDPHandler, self).__init__(**kwargs)
|
||||
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
|
||||
def _send(self, payload):
|
||||
payload = _encode_payload_for_socket(payload)
|
||||
return self.socket.sendto(payload, (self._get_host(hostname_only=True), self.port or 0))
|
||||
|
||||
@classmethod
|
||||
def perform_test(cls, settings):
|
||||
"""
|
||||
Tests logging connectivity for the current logging settings.
|
||||
"""
|
||||
handler = cls.from_django_settings(settings)
|
||||
handler.enabled_flag = True
|
||||
handler.setFormatter(LogstashFormatter(settings_module=settings))
|
||||
logger = logging.getLogger(__file__)
|
||||
fn, lno, func = logger.findCaller()
|
||||
record = logger.makeRecord('awx', 10, fn, lno,
|
||||
'AWX Connection Test', tuple(),
|
||||
None, func)
|
||||
handler.emit(_encode_payload_for_socket(record))
|
||||
self.socket.sendto(payload, (self._get_host(hostname_only=True), self.port or 0))
|
||||
return SocketResult(True, reason=self.message)
|
||||
|
||||
|
||||
HANDLER_MAPPING = {
|
||||
@@ -315,6 +264,88 @@ HANDLER_MAPPING = {
|
||||
}
|
||||
|
||||
|
||||
class AWXProxyHandler(logging.Handler):
|
||||
'''
|
||||
Handler specific to the AWX external logging feature
|
||||
|
||||
Will dynamically create a handler specific to the configured
|
||||
protocol, and will create a new one automatically on setting change
|
||||
|
||||
Managing parameters:
|
||||
All parameters will get their value from settings as a default
|
||||
if the parameter was either provided on init, or set manually,
|
||||
this value will take precedence.
|
||||
Parameters match same parameters in the actualized handler classes.
|
||||
'''
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
# TODO: process 'level' kwarg
|
||||
super(AWXProxyHandler, self).__init__(**kwargs)
|
||||
self._handler = None
|
||||
self._old_kwargs = {}
|
||||
|
||||
def get_handler_class(self, protocol):
|
||||
return HANDLER_MAPPING[protocol]
|
||||
|
||||
def get_handler(self, custom_settings=None, force_create=False):
|
||||
new_kwargs = {}
|
||||
use_settings = custom_settings or settings
|
||||
for field_name, setting_name in PARAM_NAMES.items():
|
||||
val = getattr(use_settings, setting_name, None)
|
||||
if val is None:
|
||||
continue
|
||||
new_kwargs[field_name] = val
|
||||
if new_kwargs == self._old_kwargs and self._handler and (not force_create):
|
||||
# avoids re-creating session objects, and other such things
|
||||
return self._handler
|
||||
self._old_kwargs = new_kwargs.copy()
|
||||
# TODO: remove any kwargs no applicable to that particular handler
|
||||
protocol = new_kwargs.pop('protocol', None)
|
||||
HandlerClass = self.get_handler_class(protocol)
|
||||
# cleanup old handler and make new one
|
||||
if self._handler:
|
||||
self._handler.close()
|
||||
logger.debug('Creating external log handler due to startup or settings change.')
|
||||
self._handler = HandlerClass(**new_kwargs)
|
||||
if self.formatter:
|
||||
# self.format(record) is called inside of emit method
|
||||
# so not safe to assume this can be handled within self
|
||||
self._handler.setFormatter(self.formatter)
|
||||
return self._handler
|
||||
|
||||
def emit(self, record):
|
||||
actual_handler = self.get_handler()
|
||||
return actual_handler.emit(record)
|
||||
|
||||
def perform_test(self, custom_settings):
|
||||
"""
|
||||
Tests logging connectivity for given settings module.
|
||||
@raises LoggingConnectivityException
|
||||
"""
|
||||
handler = self.get_handler(custom_settings=custom_settings, force_create=True)
|
||||
handler.setFormatter(LogstashFormatter())
|
||||
logger = logging.getLogger(__file__)
|
||||
fn, lno, func = logger.findCaller()
|
||||
record = logger.makeRecord('awx', 10, fn, lno,
|
||||
'AWX Connection Test', tuple(),
|
||||
None, func)
|
||||
futures = handler.emit(record)
|
||||
for future in futures:
|
||||
try:
|
||||
resp = future.result()
|
||||
if not resp.ok:
|
||||
if isinstance(resp, SocketResult):
|
||||
raise LoggingConnectivityException(
|
||||
'Socket error: {}'.format(resp.reason or '')
|
||||
)
|
||||
else:
|
||||
raise LoggingConnectivityException(
|
||||
': '.join([str(resp.status_code), resp.reason or ''])
|
||||
)
|
||||
except RequestException as e:
|
||||
raise LoggingConnectivityException(str(e))
|
||||
|
||||
|
||||
ColorHandler = logging.StreamHandler
|
||||
|
||||
if settings.COLOR_LOGS is True:
|
||||
@@ -340,41 +371,3 @@ if settings.COLOR_LOGS is True:
|
||||
except ImportError:
|
||||
# logutils is only used for colored logs in the dev environment
|
||||
pass
|
||||
|
||||
|
||||
def _add_or_remove_logger(address, instance):
|
||||
specific_logger = logging.getLogger(address)
|
||||
for i, handler in enumerate(specific_logger.handlers):
|
||||
if isinstance(handler, (HTTPSNullHandler, BaseHTTPSHandler)):
|
||||
specific_logger.handlers[i] = instance or HTTPSNullHandler()
|
||||
break
|
||||
else:
|
||||
if instance is not None:
|
||||
specific_logger.handlers.append(instance)
|
||||
|
||||
|
||||
def configure_external_logger(settings_module, is_startup=True):
|
||||
is_enabled = settings_module.LOG_AGGREGATOR_ENABLED
|
||||
if is_startup and (not is_enabled):
|
||||
# Pass-through if external logging not being used
|
||||
return
|
||||
|
||||
instance = None
|
||||
if is_enabled:
|
||||
handler_class = HANDLER_MAPPING[settings_module.LOG_AGGREGATOR_PROTOCOL]
|
||||
instance = handler_class.from_django_settings(settings_module)
|
||||
|
||||
# Obtain the Formatter class from settings to maintain customizations
|
||||
configurator = logging.config.DictConfigurator(settings_module.LOGGING)
|
||||
formatter_config = settings_module.LOGGING['formatters']['json'].copy()
|
||||
formatter_config['settings_module'] = settings_module
|
||||
formatter = configurator.configure_custom(formatter_config)
|
||||
|
||||
instance.setFormatter(formatter)
|
||||
|
||||
awx_logger_instance = instance
|
||||
if is_enabled and 'awx' not in settings_module.LOG_AGGREGATOR_LOGGERS:
|
||||
awx_logger_instance = None
|
||||
|
||||
_add_or_remove_logger('awx.analytics', instance)
|
||||
_add_or_remove_logger('awx', awx_logger_instance)
|
||||
|
||||
Reference in New Issue
Block a user