From 017d367749f69e13a334e378eff3446eeea850ae Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 19 Nov 2018 07:56:27 -0500 Subject: [PATCH 1/4] Remove dependency and insert class --- awx/main/utils/formatters.py | 82 ++++++++++++++++++++++++++++++++++- requirements/requirements.in | 1 - requirements/requirements.txt | 2 +- 3 files changed, 81 insertions(+), 4 deletions(-) diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py index c7beee6b0c..7b8f06d535 100644 --- a/awx/main/utils/formatters.py +++ b/awx/main/utils/formatters.py @@ -1,11 +1,14 @@ # Copyright (c) 2017 Ansible Tower by Red Hat # All Rights Reserved. -from logstash.formatter import LogstashFormatterVersion1 from copy import copy import json import time import logging +import traceback +import socket +import sys +from datetime import datetime from django.conf import settings @@ -20,7 +23,82 @@ class TimeFormatter(logging.Formatter): return logging.Formatter.format(self, record) -class LogstashFormatter(LogstashFormatterVersion1): +class LogstashFormatterBase(logging.Formatter): + + def __init__(self, message_type='Logstash', tags=None, fqdn=False): + self.message_type = message_type + self.tags = tags if tags is not None else [] + + if fqdn: + self.host = socket.getfqdn() + else: + self.host = socket.gethostname() + + def get_extra_fields(self, record): + # The list contains all the attributes listed in + # http://docs.python.org/library/logging.html#logrecord-attributes + skip_list = ( + 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', + 'funcName', 'id', 'levelname', 'levelno', 'lineno', 'module', + 'msecs', 'msecs', 'message', 'msg', 'name', 'pathname', 'process', + 'processName', 'relativeCreated', 'thread', 'threadName', 'extra') + + if sys.version_info < (3, 0): + easy_types = (basestring, bool, dict, float, int, long, list, type(None)) + else: + easy_types = (str, bool, dict, float, int, list, type(None)) + + fields = {} + + for key, value in record.__dict__.items(): + if key not in skip_list: + if isinstance(value, easy_types): + fields[key] = value + else: + fields[key] = repr(value) + + return fields + + def get_debug_fields(self, record): + fields = { + 'stack_trace': self.format_exception(record.exc_info), + 'lineno': record.lineno, + 'process': record.process, + 'thread_name': record.threadName, + } + + # funcName was added in 2.5 + if not getattr(record, 'funcName', None): + fields['funcName'] = record.funcName + + # processName was added in 2.6 + if not getattr(record, 'processName', None): + fields['processName'] = record.processName + + return fields + + @classmethod + def format_source(cls, message_type, host, path): + return "%s://%s/%s" % (message_type, host, path) + + @classmethod + def format_timestamp(cls, time): + tstamp = datetime.utcfromtimestamp(time) + return tstamp.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (tstamp.microsecond / 1000) + "Z" + + @classmethod + def format_exception(cls, exc_info): + return ''.join(traceback.format_exception(*exc_info)) if exc_info else '' + + @classmethod + def serialize(cls, message): + if sys.version_info < (3, 0): + return json.dumps(message) + else: + return bytes(json.dumps(message), 'utf-8') + + +class LogstashFormatter(LogstashFormatterBase): def reformat_data_for_log(self, raw_data, kind=None): ''' diff --git a/requirements/requirements.in b/requirements/requirements.in index 04b68a511c..eb1731ea9b 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -32,7 +32,6 @@ psycopg2==2.7.3.2 # problems with Segmentation faults / wheels on upgrade pygerduty==0.37.0 pyparsing==2.2.0 python-dateutil==2.7.2 # contains support for TZINFO= parsing -python-logstash==0.4.6 python-memcached==1.59 python-radius==1.0 python3-saml==1.4.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index eedd26c369..12472b93af 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -80,7 +80,7 @@ pyrad==2.1 # via django-radius pysocks==1.6.8 # via twilio python-dateutil==2.7.2 python-ldap==3.1.0 # via django-auth-ldap -python-logstash==0.4.6 +django-cors-headers==2.4.0 python-memcached==1.59 python-radius==1.0 python3-openid==3.1.0 # via social-auth-core From 127495b53d60e90fb06543be3fa4e5cb218c1bb6 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 19 Nov 2018 08:03:36 -0500 Subject: [PATCH 2/4] remove things from base class that were never used --- awx/main/utils/formatters.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py index 7b8f06d535..3711cf33a8 100644 --- a/awx/main/utils/formatters.py +++ b/awx/main/utils/formatters.py @@ -24,10 +24,13 @@ class TimeFormatter(logging.Formatter): class LogstashFormatterBase(logging.Formatter): + """Base class taken from python-logstash=0.4.6 + modified here since that version + License in docs/licenses/ + """ - def __init__(self, message_type='Logstash', tags=None, fqdn=False): + def __init__(self, message_type='Logstash', fqdn=False): self.message_type = message_type - self.tags = tags if tags is not None else [] if fqdn: self.host = socket.getfqdn() @@ -77,10 +80,6 @@ class LogstashFormatterBase(logging.Formatter): return fields - @classmethod - def format_source(cls, message_type, host, path): - return "%s://%s/%s" % (message_type, host, path) - @classmethod def format_timestamp(cls, time): tstamp = datetime.utcfromtimestamp(time) @@ -234,10 +233,8 @@ class LogstashFormatter(LogstashFormatterBase): def format(self, record): message = { - # Fields not included, but exist in related logs + # Field not included, but exist in related logs # 'path': record.pathname - # '@version': '1', # from python-logstash - # 'tags': self.tags, '@timestamp': self.format_timestamp(record.created), 'message': record.getMessage(), 'host': self.host, From 8a72a4d39d70d278c0be49620ac383c0f17ecef8 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 23 Jan 2019 09:48:51 -0500 Subject: [PATCH 3/4] Prune the python2 specific logic from log formatter --- awx/main/utils/formatters.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py index 3711cf33a8..9d1a431b0b 100644 --- a/awx/main/utils/formatters.py +++ b/awx/main/utils/formatters.py @@ -46,10 +46,7 @@ class LogstashFormatterBase(logging.Formatter): 'msecs', 'msecs', 'message', 'msg', 'name', 'pathname', 'process', 'processName', 'relativeCreated', 'thread', 'threadName', 'extra') - if sys.version_info < (3, 0): - easy_types = (basestring, bool, dict, float, int, long, list, type(None)) - else: - easy_types = (str, bool, dict, float, int, list, type(None)) + easy_types = (str, bool, dict, float, int, list, type(None)) fields = {} @@ -63,23 +60,15 @@ class LogstashFormatterBase(logging.Formatter): return fields def get_debug_fields(self, record): - fields = { + return { 'stack_trace': self.format_exception(record.exc_info), 'lineno': record.lineno, 'process': record.process, 'thread_name': record.threadName, + 'funcName': record.funcName, + 'processName': record.processName, } - # funcName was added in 2.5 - if not getattr(record, 'funcName', None): - fields['funcName'] = record.funcName - - # processName was added in 2.6 - if not getattr(record, 'processName', None): - fields['processName'] = record.processName - - return fields - @classmethod def format_timestamp(cls, time): tstamp = datetime.utcfromtimestamp(time) @@ -91,10 +80,7 @@ class LogstashFormatterBase(logging.Formatter): @classmethod def serialize(cls, message): - if sys.version_info < (3, 0): - return json.dumps(message) - else: - return bytes(json.dumps(message), 'utf-8') + return bytes(json.dumps(message), 'utf-8') class LogstashFormatter(LogstashFormatterBase): From 1ae4ed492290b85fab00f786c9125ab297bce1f7 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 23 Jan 2019 10:15:30 -0500 Subject: [PATCH 4/4] migrate python-logstash license --- awx/main/utils/formatters.py | 26 ++++++++++++++++++++++++-- docs/licenses/python-logstash.txt | 21 --------------------- requirements/requirements.txt | 1 - 3 files changed, 24 insertions(+), 24 deletions(-) delete mode 100644 docs/licenses/python-logstash.txt diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py index 9d1a431b0b..6009c03157 100644 --- a/awx/main/utils/formatters.py +++ b/awx/main/utils/formatters.py @@ -7,7 +7,6 @@ import time import logging import traceback import socket -import sys from datetime import datetime @@ -26,7 +25,30 @@ class TimeFormatter(logging.Formatter): class LogstashFormatterBase(logging.Formatter): """Base class taken from python-logstash=0.4.6 modified here since that version - License in docs/licenses/ + + For compliance purposes, this was the license at the point of divergence: + + The MIT License (MIT) + + Copyright (c) 2013, Volodymyr Klochan + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ def __init__(self, message_type='Logstash', fqdn=False): diff --git a/docs/licenses/python-logstash.txt b/docs/licenses/python-logstash.txt deleted file mode 100644 index 00339c895c..0000000000 --- a/docs/licenses/python-logstash.txt +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013, Volodymyr Klochan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 12472b93af..2b384178a0 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -80,7 +80,6 @@ pyrad==2.1 # via django-radius pysocks==1.6.8 # via twilio python-dateutil==2.7.2 python-ldap==3.1.0 # via django-auth-ldap -django-cors-headers==2.4.0 python-memcached==1.59 python-radius==1.0 python3-openid==3.1.0 # via social-auth-core