From 2a4b009f04ae7d29e111ce27de21201b37076ae4 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 9 Apr 2020 00:42:43 -0400 Subject: [PATCH] rsyslogd: use %rawmsg-after-pri% instead of %msg% after some prolonged RFC reading and tinkering w/ rsyslogd... cpython's SysLogHandler doesn't emit RFC3164 formatted messages in the format you'd expect; it's missing the ISO date, hostname, etc... along with other header values; the handler implementation relies on you to specify a syslog-like formatter (we've replaced all of this with our own *custom* logstash-esque formatter that effectively outputs valid JSON - without dates and other syslog header values prepended) because of this unanticipated format, rsyslogd chokes when trying to parse the message's parts; AWX is emitting: RAWJSON ...so the usage of `%msg%` isn't going to work for us, because rsyslog tries to parse *all* of the possible headers (and yells, because it can't find a date to parse): see: https://www.rsyslog.com/files/temp/doc-indent/configuration/properties.html#message-properties this is fine, because we don't *need* any of that message parsing anyways; in the end, we're *just* interested in forwarding the raw JSON/text content to the third party log handler --- awx/main/tests/unit/api/test_logger.py | 18 +++++++++--------- awx/main/utils/external_logging.py | 2 +- awx/main/utils/formatters.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/awx/main/tests/unit/api/test_logger.py b/awx/main/tests/unit/api/test_logger.py index ac1a63acbb..17408a9421 100644 --- a/awx/main/tests/unit/api/test_logger.py +++ b/awx/main/tests/unit/api/test_logger.py @@ -43,7 +43,7 @@ data_loggly = { None, 'https', '\n'.join([ - 'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', 'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")', # noqa ]) ), @@ -54,7 +54,7 @@ data_loggly = { 9000, 'udp', '\n'.join([ - 'template(name="awx" type="string" string="%msg%")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")', 'action(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" template="awx")', # noqa ]) ), @@ -65,7 +65,7 @@ data_loggly = { 9000, 'tcp', '\n'.join([ - 'template(name="awx" type="string" string="%msg%")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")', 'action(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" template="awx")', # noqa ]) ), @@ -76,7 +76,7 @@ data_loggly = { None, None, '\n'.join([ - 'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', 'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")', # noqa ]) ), @@ -87,7 +87,7 @@ data_loggly = { None, None, '\n'.join([ - 'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', 'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")', # noqa ]) ), @@ -98,7 +98,7 @@ data_loggly = { None, None, '\n'.join([ - 'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', 'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")', # noqa ]) ), @@ -109,7 +109,7 @@ data_loggly = { 8088, None, '\n'.join([ - 'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', 'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")', # noqa ]) ), @@ -120,7 +120,7 @@ data_loggly = { 8088, 'https', '\n'.join([ - 'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', 'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")', # noqa ]) ), @@ -131,7 +131,7 @@ data_loggly = { 8088, None, '\n'.join([ - 'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', 'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")', # noqa ]) ), diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py index 32f90af8d8..11ea0cf53f 100644 --- a/awx/main/utils/external_logging.py +++ b/awx/main/utils/external_logging.py @@ -32,7 +32,7 @@ def construct_rsyslog_conf_template(settings=settings): '$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf', '$ModLoad imuxsock', 'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")', - 'template(name="awx" type="string" string="%msg%")', + 'template(name="awx" type="string" string="%rawmsg-after-pri%")', ]) if protocol.startswith('http'): # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py index fb08034a5e..8e3ddabf1b 100644 --- a/awx/main/utils/formatters.py +++ b/awx/main/utils/formatters.py @@ -97,7 +97,7 @@ class LogstashFormatterBase(logging.Formatter): @classmethod def serialize(cls, message): - return ' ' + json.dumps(message, cls=DjangoJSONEncoder) + '\000' + return json.dumps(message, cls=DjangoJSONEncoder) + '\n' class LogstashFormatter(LogstashFormatterBase):