From d3f047d731f20db0f38a6dd0fbf80bd25ef244b0 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 22 May 2019 09:46:19 -0400 Subject: [PATCH] Move dynamic log level logic to filter --- awx/main/utils/filters.py | 45 +++++++++++++++++++++++++++++--------- awx/main/utils/handlers.py | 27 ++--------------------- awx/settings/defaults.py | 21 ++++++++++-------- 3 files changed, 49 insertions(+), 44 deletions(-) diff --git a/awx/main/utils/filters.py b/awx/main/utils/filters.py index 015bfd8901..2edee2b66a 100644 --- a/awx/main/utils/filters.py +++ b/awx/main/utils/filters.py @@ -18,7 +18,7 @@ from django.conf import settings from awx.main.constants import LOGGER_BLACKLIST from awx.main.utils.common import get_search_fields -__all__ = ['SmartFilter', 'ExternalLoggerEnabled'] +__all__ = ['SmartFilter', 'ExternalLoggerEnabled', 'DynamicLevelFilter'] logger = logging.getLogger('awx.main.utils') @@ -48,9 +48,18 @@ class FieldFromSettings(object): instance.settings_override[self.setting_name] = value +def record_is_blacklisted(record): + """Given a log record, return True if it is considered to be in + the logging blacklist, return False if not + """ + for logger_name in LOGGER_BLACKLIST: + if record.name.startswith(logger_name): + return True + return False + + class ExternalLoggerEnabled(Filter): - lvl = FieldFromSettings('LOG_AGGREGATOR_LEVEL') enabled_loggers = FieldFromSettings('LOG_AGGREGATOR_LOGGERS') enabled_flag = FieldFromSettings('LOG_AGGREGATOR_ENABLED') @@ -64,18 +73,16 @@ class ExternalLoggerEnabled(Filter): setattr(self, field_name, field_value) def filter(self, record): - """ - Uses the database settings to determine if the current - external log configuration says that this particular record - should be sent to the external log aggregator + """Filters out all log records if LOG_AGGREGATOR_ENABLED is False + or if the particular logger is not in LOG_AGGREGATOR_LOGGERS + should only be used for the external logger False - should not be logged True - should be logged """ - # Logger exceptions - for logger_name in LOGGER_BLACKLIST: - if record.name.startswith(logger_name): - return False + # Do not send exceptions to external logger + if record_is_blacklisted(record): + return False # General enablement if not self.enabled_flag: return False @@ -95,6 +102,24 @@ class ExternalLoggerEnabled(Filter): return bool(base_name in loggers) +class DynamicLevelFilter(Filter): + + def filter(self, record): + """Filters out logs that have a level below the threshold defined + by the databse setting LOG_AGGREGATOR_LEVEL + """ + if record_is_blacklisted(record): + # Fine to write blacklisted loggers to file, apply default filtering level + cutoff_level = logging.WARNING + else: + try: + cutoff_level = logging._nameToLevel[settings.LOG_AGGREGATOR_LEVEL] + except Exception: + cutoff_level = logging.WARNING + + return bool(record.levelno >= cutoff_level) + + def string_to_type(t): if t == u'null': return None diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index 17f94f799d..aac46966b5 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -19,12 +19,11 @@ from django.conf import settings from requests_futures.sessions import FuturesSession # AWX -from awx.main.constants import LOGGER_BLACKLIST from awx.main.utils.formatters import LogstashFormatter __all__ = ['BaseHTTPSHandler', 'TCPHandler', 'UDPHandler', - 'AWXProxyHandler', 'RotatingProductionLogHandler'] + 'AWXProxyHandler'] logger = logging.getLogger('awx.main.utils.handlers') @@ -92,28 +91,6 @@ class SocketResult: return self -class DynamicLevelMixin(object): - - @property - def level(self): - from django.conf import settings - for logger_name in LOGGER_BLACKLIST: - if self.name.startswith(logger_name): - return 30 # WARNING - try: - return logging._nameToLevel[settings.LOG_AGGREGATOR_LEVEL] - except Exception: - return 30 # WARNING - - @level.setter - def level(self, level): - pass # no-op, this value comes from the database - - -class RotatingProductionLogHandler(logging.handlers.RotatingFileHandler, DynamicLevelMixin): - pass - - class BaseHandler(logging.Handler): def __init__(self, host=None, port=None, indv_facts=None, **kwargs): super(BaseHandler, self).__init__() @@ -295,7 +272,7 @@ HANDLER_MAPPING = { } -class AWXProxyHandler(logging.Handler, DynamicLevelMixin): +class AWXProxyHandler(logging.Handler): ''' Handler specific to the AWX external logging feature diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 980672fca2..e72da9a8bd 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -966,6 +966,9 @@ LOGGING = { 'external_log_enabled': { '()': 'awx.main.utils.filters.ExternalLoggerEnabled' }, + 'dynamic_level_filter': { + '()': 'awx.main.utils.filters.DynamicLevelFilter' + } }, 'formatters': { 'simple': { @@ -1005,7 +1008,7 @@ LOGGING = { 'external_logger': { 'class': 'awx.main.utils.handlers.AWXProxyHandler', 'formatter': 'json', - 'filters': ['external_log_enabled'], + 'filters': ['external_log_enabled', 'dynamic_level_filter'], }, 'mail_admins': { 'level': 'ERROR', @@ -1014,8 +1017,8 @@ LOGGING = { }, 'tower_warnings': { # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL - 'class':'awx.main.utils.handlers.RotatingProductionLogHandler', - 'filters': ['require_debug_false'], + 'class': 'logging.handlers.RotatingFileHandler', + 'filters': ['require_debug_false', 'dynamic_level_filter'], 'filename': os.path.join(LOG_ROOT, 'tower.log'), 'maxBytes': 1024 * 1024 * 5, # 5 MB 'backupCount': 5, @@ -1023,8 +1026,8 @@ LOGGING = { }, 'callback_receiver': { # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL - 'class':'awx.main.utils.handlers.RotatingProductionLogHandler', - 'filters': ['require_debug_false'], + 'class': 'logging.handlers.RotatingFileHandler', + 'filters': ['require_debug_false', 'dynamic_level_filter'], 'filename': os.path.join(LOG_ROOT, 'callback_receiver.log'), 'maxBytes': 1024 * 1024 * 5, # 5 MB 'backupCount': 5, @@ -1032,8 +1035,8 @@ LOGGING = { }, 'dispatcher': { # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL - 'class':'awx.main.utils.handlers.RotatingProductionLogHandler', - 'filters': ['require_debug_false'], + 'class': 'logging.handlers.RotatingFileHandler', + 'filters': ['require_debug_false', 'dynamic_level_filter'], 'filename': os.path.join(LOG_ROOT, 'dispatcher.log'), 'maxBytes': 1024 * 1024 * 5, # 5 MB 'backupCount': 5, @@ -1050,8 +1053,8 @@ LOGGING = { }, 'task_system': { # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL - 'class':'awx.main.utils.handlers.RotatingProductionLogHandler', - 'filters': ['require_debug_false'], + 'class': 'logging.handlers.RotatingFileHandler', + 'filters': ['require_debug_false', 'dynamic_level_filter'], 'filename': os.path.join(LOG_ROOT, 'task_system.log'), 'maxBytes': 1024 * 1024 * 5, # 5 MB 'backupCount': 5,