Ensure log messages have valid json

- Fix messages getting contatenated at 8k
 - Fix rsyslog cutting off the opening brace of log messages
 - Make valid default conf and emit logs based on prescence of .sock and
 settings

awx/main/utils/external_logging.py

@@ -8,65 +8,66 @@ from awx.main.utils.reload import supervisor_service_command
 def construct_rsyslog_conf_template(settings=settings):
     tmpl = ''
     parts = []
-    if settings.LOG_AGGREGATOR_ENABLED:
+    host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
-        host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
+    port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
-        port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
+    protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '')
-        protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '')
+    if protocol.startswith('http'):
-        if protocol.startswith('http'):
+        scheme = 'https'
-            scheme = 'https'
+        # urlparse requires '//' to be provided if scheme is not specified
-            # urlparse requires '//' to be provided if scheme is not specified
+        original_parsed = urlparse.urlsplit(host)
-            original_parsed = urlparse.urlsplit(host)
+        if (not original_parsed.scheme and not host.startswith('//')) or original_parsed.hostname is None:
-            if (not original_parsed.scheme and not host.startswith('//')) or original_parsed.hostname is None:
+            host = '%s://%s' % (scheme, host) if scheme else '//%s' % host
-                host = '%s://%s' % (scheme, host) if scheme else '//%s' % host
+        parsed = urlparse.urlsplit(host)
-            parsed = urlparse.urlsplit(host)
 
-            host = parsed.hostname
+        host = parsed.hostname
-            try:
+        try:
-                if parsed.port:
+            if parsed.port:
-                    port = parsed.port
+                port = parsed.port
-            except ValueError:
+        except ValueError:
-                port = settings.LOG_AGGREGATOR_PORT
+            port = settings.LOG_AGGREGATOR_PORT
-        max_bytes = settings.MAX_EVENT_RES_DATA
+    
-        parts.extend([
+    max_bytes = settings.MAX_EVENT_RES_DATA
-            '$WorkDirectory /var/lib/awx/rsyslog',
+    parts.extend([
-            '$IncludeConfig /etc/rsyslog.d/*.conf',
+        '$WorkDirectory /var/lib/awx/rsyslog',
-            f'$MaxMessageSize {max_bytes}b',
+        f'$MaxMessageSize {max_bytes}',
-            '$ModLoad imuxsock',
+        '$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf',
-            'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")',
+        '$ModLoad imuxsock',
-            'template(name="awx" type="string" string="%msg%")',
+        'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")',
-        ])
+        'template(name="awx" type="string" string="%msg%")',
-        if protocol.startswith('http'):
+    ])
-            # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst
+    if protocol.startswith('http'):
-            ssl = "on" if parsed.scheme == 'https' else "off"
+        # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst
-            skip_verify = "off" if settings.LOG_AGGREGATOR_VERIFY_CERT else "on"
+        ssl = "on" if parsed.scheme == 'https' else "off"
-            if not port:
+        skip_verify = "off" if settings.LOG_AGGREGATOR_VERIFY_CERT else "on"
-                port = 443 if parsed.scheme == 'https' else 80
+        if not port:
+            port = 443 if parsed.scheme == 'https' else 80
 
-            params = [
+        params = [
-                'type="omhttp"',
+            'type="omhttp"',
-                f'server="{host}"',
+            f'server="{host}"',
-                f'serverport="{port}"',
+            f'serverport="{port}"',
-                f'usehttps="{ssl}"',
+            f'usehttps="{ssl}"',
-                f'skipverifyhost="{skip_verify}"',
+            f'skipverifyhost="{skip_verify}"',
-                'action.resumeRetryCount="-1"',
+            'action.resumeRetryCount="-1"',
-                'template="awx"',
+            'template="awx"',
-                'errorfile="/var/log/tower/external.err"',
+            'errorfile="/var/log/tower/external.err"',
-                'healthchecktimeout="20000"',
+            'healthchecktimeout="20000"',
-            ]
+        ]
-            if parsed.path:
+        if parsed.path:
-                params.append(f'restpath="{parsed.path[1:]}"')
+            params.append(f'restpath="{parsed.path[1:]}"')
-            username = getattr(settings, 'LOG_AGGREGATOR_USERNAME', '')
+        username = getattr(settings, 'LOG_AGGREGATOR_USERNAME', '')
-            password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '')
+        password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '')
-            if username:
+        if username:
-                params.append(f'uid="{username}"')
+            params.append(f'uid="{username}"')
-            if password:
+        if password:
-                params.append(f'pwd="{password}"')
+            params.append(f'pwd="{password}"')
-            params = ' '.join(params)
+        params = ' '.join(params)
-            parts.extend(['module(load="omhttp")', f'action({params})'])
+        parts.extend(['module(load="omhttp")', f'action({params})'])
-        else:
+    elif protocol and host and port:
-            parts.append(
+        parts.append(
-                f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
+            f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
-            )
+        )
+    # parts.append('$IncludeConfig /var/lib/awx/rsyslog/*.conf')
     tmpl = '\n'.join(parts)
     return tmpl
 

awx/main/utils/handlers.py

@@ -14,6 +14,8 @@ class RSysLogHandler(logging.handlers.SysLogHandler):
     append_nul = False
 
     def emit(self, msg):
+        if not settings.LOG_AGGREGATOR_ENABLED:
+            return
         if not os.path.exists(settings.LOGGING['handlers']['external_logger']['address']):
             return
         return super(RSysLogHandler, self).emit(msg)

installer/roles/image_build/files/rsyslog.conf

@@ -1 +1,6 @@
 $WorkDirectory /var/lib/awx/rsyslog
+$MaxMessageSize 700000
+$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf
+$ModLoad imuxsock
+input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")

installer/roles/image_build/templates/Dockerfile.j2

@@ -101,7 +101,7 @@ ADD rsyslog.repo /etc/yum.repos.d/
 RUN yum install -y rsyslog-omhttp
 
 # Pre-create things that we need to write to
-RUN for dir in /home/awx /var/run/supervisor /var/lib/awx /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \
+RUN for dir in /home/awx /var/run/supervisor /var/lib/awx /var/lib/awx/rsyslog /var/lib/awx/rsyslog/conf.d /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /var/run/nginx.pid; \

tools/docker-compose/Dockerfile

@@ -123,7 +123,7 @@ ADD tools/docker-compose/entrypoint.sh /
 ADD tools/scripts/awx-python /usr/bin/awx-python
 
 # Pre-create things that we need to write to
-RUN for dir in /var/lib/awx /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
+RUN for dir in /var/lib/awx /var/lib/awx/rsyslog /var/lib/awx/rsyslog/conf.d /var/run/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \

tools/docker-compose/rsyslog.conf

@@ -1 +1,6 @@
 $WorkDirectory /var/lib/awx/rsyslog
+$MaxMessageSize 700000
+$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf
+$ModLoad imuxsock
+input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")