Merge pull request #9452 from ryanpetrello/rsyslogd-last-ditch

make rsyslogd socket emit failures a bit less verbose (but still write to stderr)

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul
This commit is contained in:
softwarefactory-project-zuul[bot] 2021-03-05 19:14:33 +00:00 committed by GitHub
commit 2ae82b9302
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 25 additions and 20 deletions

View File

@ -32,7 +32,7 @@ def construct_rsyslog_conf_template(settings=settings):
'$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf',
f'main_queue(queue.spoolDirectory="{spool_directory}" queue.maxdiskspace="{max_disk_space}g" queue.type="Disk" queue.filename="awx-external-logger-backlog")', # noqa
'module(load="imuxsock" SysSock.Use="off")',
'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")',
'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on" RateLimit.Burst="0")',
'template(name="awx" type="string" string="%rawmsg-after-pri%")',
])

View File

@ -3,7 +3,8 @@
# Python
import logging
import os.path
import sys
import traceback
# Django
from django.conf import settings
@ -21,27 +22,31 @@ class RSysLogHandler(logging.handlers.SysLogHandler):
super(RSysLogHandler, self)._connect_unixsocket(address)
self.socket.setblocking(False)
def handleError(self, record):
# for any number of reasons, rsyslogd has gone to lunch;
# this usually means that it's just been restarted (due to
# a configuration change) unfortunately, we can't log that
# because...rsyslogd is down (and would just put us back down this
# code path)
# as a fallback, it makes the most sense to just write the
# messages to sys.stderr (which will end up in supervisord logs,
# and in containerized installs, cascaded down to pod logs)
# because the alternative is blocking the
# socket.send() in the Python process, which we definitely don't
# want to do)
msg = f'{record.asctime} ERROR rsyslogd was unresponsive: '
exc = traceback.format_exc()
try:
msg += exc.splitlines()[-1]
except Exception:
msg += exc
msg = '\n'.join([msg, record.msg, ''])
sys.stderr.write(msg)
def emit(self, msg):
if not settings.LOG_AGGREGATOR_ENABLED:
return
if not os.path.exists(settings.LOGGING['handlers']['external_logger']['address']):
return
try:
return super(RSysLogHandler, self).emit(msg)
except ConnectionRefusedError:
# rsyslogd has gone to lunch; this generally means that it's just
# been restarted (due to a configuration change)
# unfortunately, we can't log that because...rsyslogd is down (and
# would just us back ddown this code path)
pass
except BlockingIOError:
# for <some reason>, rsyslogd is no longer reading from the domain socket, and
# we're unable to write any more to it without blocking (we've seen this behavior
# from time to time when logging is totally misconfigured;
# in this scenario, it also makes more sense to just drop the messages,
# because the alternative is blocking the socket.send() in the
# Python process, which we definitely don't want to do)
pass
return super(RSysLogHandler, self).emit(msg)
class SpecialInventoryHandler(logging.Handler):