From b942fde59a5536089d046abfe66d655d549f8c08 Mon Sep 17 00:00:00 2001 From: Christian Adams Date: Wed, 8 Apr 2020 14:37:21 -0400 Subject: [PATCH] Ensure log messages have valid json - Fix messages getting contatenated at 8k - Fix rsyslog cutting off the opening brace of log messages - Make valid default conf and emit logs based on prescence of .sock and settings --- awx/main/utils/external_logging.py | 115 +++++++++--------- awx/main/utils/handlers.py | 2 + .../roles/image_build/files/rsyslog.conf | 5 + .../roles/image_build/templates/Dockerfile.j2 | 2 +- tools/docker-compose/Dockerfile | 2 +- tools/docker-compose/rsyslog.conf | 5 + 6 files changed, 72 insertions(+), 59 deletions(-) diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py index a8295ad399..e5e11d5156 100644 --- a/awx/main/utils/external_logging.py +++ b/awx/main/utils/external_logging.py @@ -8,65 +8,66 @@ from awx.main.utils.reload import supervisor_service_command def construct_rsyslog_conf_template(settings=settings): tmpl = '' parts = [] - if settings.LOG_AGGREGATOR_ENABLED: - host = getattr(settings, 'LOG_AGGREGATOR_HOST', '') - port = getattr(settings, 'LOG_AGGREGATOR_PORT', '') - protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '') - if protocol.startswith('http'): - scheme = 'https' - # urlparse requires '//' to be provided if scheme is not specified - original_parsed = urlparse.urlsplit(host) - if (not original_parsed.scheme and not host.startswith('//')) or original_parsed.hostname is None: - host = '%s://%s' % (scheme, host) if scheme else '//%s' % host - parsed = urlparse.urlsplit(host) + host = getattr(settings, 'LOG_AGGREGATOR_HOST', '') + port = getattr(settings, 'LOG_AGGREGATOR_PORT', '') + protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '') + if protocol.startswith('http'): + scheme = 'https' + # urlparse requires '//' to be provided if scheme is not specified + original_parsed = urlparse.urlsplit(host) + if (not original_parsed.scheme and not host.startswith('//')) or original_parsed.hostname is None: + host = '%s://%s' % (scheme, host) if scheme else '//%s' % host + parsed = urlparse.urlsplit(host) - host = parsed.hostname - try: - if parsed.port: - port = parsed.port - except ValueError: - port = settings.LOG_AGGREGATOR_PORT - max_bytes = settings.MAX_EVENT_RES_DATA - parts.extend([ - '$WorkDirectory /var/lib/awx/rsyslog', - '$IncludeConfig /etc/rsyslog.d/*.conf', - f'$MaxMessageSize {max_bytes}b', - '$ModLoad imuxsock', - 'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")', - 'template(name="awx" type="string" string="%msg%")', - ]) - if protocol.startswith('http'): - # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst - ssl = "on" if parsed.scheme == 'https' else "off" - skip_verify = "off" if settings.LOG_AGGREGATOR_VERIFY_CERT else "on" - if not port: - port = 443 if parsed.scheme == 'https' else 80 + host = parsed.hostname + try: + if parsed.port: + port = parsed.port + except ValueError: + port = settings.LOG_AGGREGATOR_PORT + + max_bytes = settings.MAX_EVENT_RES_DATA + parts.extend([ + '$WorkDirectory /var/lib/awx/rsyslog', + f'$MaxMessageSize {max_bytes}', + '$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf', + '$ModLoad imuxsock', + 'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")', + 'template(name="awx" type="string" string="%msg%")', + ]) + if protocol.startswith('http'): + # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst + ssl = "on" if parsed.scheme == 'https' else "off" + skip_verify = "off" if settings.LOG_AGGREGATOR_VERIFY_CERT else "on" + if not port: + port = 443 if parsed.scheme == 'https' else 80 - params = [ - 'type="omhttp"', - f'server="{host}"', - f'serverport="{port}"', - f'usehttps="{ssl}"', - f'skipverifyhost="{skip_verify}"', - 'action.resumeRetryCount="-1"', - 'template="awx"', - 'errorfile="/var/log/tower/external.err"', - 'healthchecktimeout="20000"', - ] - if parsed.path: - params.append(f'restpath="{parsed.path[1:]}"') - username = getattr(settings, 'LOG_AGGREGATOR_USERNAME', '') - password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '') - if username: - params.append(f'uid="{username}"') - if password: - params.append(f'pwd="{password}"') - params = ' '.join(params) - parts.extend(['module(load="omhttp")', f'action({params})']) - else: - parts.append( - f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")' # noqa - ) + params = [ + 'type="omhttp"', + f'server="{host}"', + f'serverport="{port}"', + f'usehttps="{ssl}"', + f'skipverifyhost="{skip_verify}"', + 'action.resumeRetryCount="-1"', + 'template="awx"', + 'errorfile="/var/log/tower/external.err"', + 'healthchecktimeout="20000"', + ] + if parsed.path: + params.append(f'restpath="{parsed.path[1:]}"') + username = getattr(settings, 'LOG_AGGREGATOR_USERNAME', '') + password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '') + if username: + params.append(f'uid="{username}"') + if password: + params.append(f'pwd="{password}"') + params = ' '.join(params) + parts.extend(['module(load="omhttp")', f'action({params})']) + elif protocol and host and port: + parts.append( + f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")' # noqa + ) + # parts.append('$IncludeConfig /var/lib/awx/rsyslog/*.conf') tmpl = '\n'.join(parts) return tmpl diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index f7c11cbc53..ebf299a9f0 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -14,6 +14,8 @@ class RSysLogHandler(logging.handlers.SysLogHandler): append_nul = False def emit(self, msg): + if not settings.LOG_AGGREGATOR_ENABLED: + return if not os.path.exists(settings.LOGGING['handlers']['external_logger']['address']): return return super(RSysLogHandler, self).emit(msg) diff --git a/installer/roles/image_build/files/rsyslog.conf b/installer/roles/image_build/files/rsyslog.conf index cc67dc4ebd..c3a2d6727d 100644 --- a/installer/roles/image_build/files/rsyslog.conf +++ b/installer/roles/image_build/files/rsyslog.conf @@ -1 +1,6 @@ $WorkDirectory /var/lib/awx/rsyslog +$MaxMessageSize 700000 +$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf +$ModLoad imuxsock +input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on") +template(name="awx" type="string" string="%msg%") diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2 index 3834ce3a76..50ec4686e1 100644 --- a/installer/roles/image_build/templates/Dockerfile.j2 +++ b/installer/roles/image_build/templates/Dockerfile.j2 @@ -101,7 +101,7 @@ ADD rsyslog.repo /etc/yum.repos.d/ RUN yum install -y rsyslog-omhttp # Pre-create things that we need to write to -RUN for dir in /home/awx /var/run/supervisor /var/lib/awx /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \ +RUN for dir in /home/awx /var/run/supervisor /var/lib/awx /var/lib/awx/rsyslog /var/lib/awx/rsyslog/conf.d /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \ do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \ \ for file in /etc/passwd /var/run/nginx.pid; \ diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile index 62536d7b18..88d0af0bf4 100644 --- a/tools/docker-compose/Dockerfile +++ b/tools/docker-compose/Dockerfile @@ -123,7 +123,7 @@ ADD tools/docker-compose/entrypoint.sh / ADD tools/scripts/awx-python /usr/bin/awx-python # Pre-create things that we need to write to -RUN for dir in /var/lib/awx /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \ +RUN for dir in /var/lib/awx /var/lib/awx/rsyslog /var/lib/awx/rsyslog/conf.d /var/run/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \ do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \ \ for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \ diff --git a/tools/docker-compose/rsyslog.conf b/tools/docker-compose/rsyslog.conf index cc67dc4ebd..c3a2d6727d 100644 --- a/tools/docker-compose/rsyslog.conf +++ b/tools/docker-compose/rsyslog.conf @@ -1 +1,6 @@ $WorkDirectory /var/lib/awx/rsyslog +$MaxMessageSize 700000 +$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf +$ModLoad imuxsock +input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on") +template(name="awx" type="string" string="%msg%")