Configure rsyslog to listen over a unix domain socket instead of a port

- Add a placeholder rsyslog.conf so it doesn't fail on start
 - Create access restricted directory for unix socket to be created in
 - Create RSyslogHandler to exit early when logging socket doesn't exist
 - Write updated logging settings when dispatcher comes up and restart rsyslog so they  take effect
 - Move rsyslogd to the web container and create rpc supervisor.sock
 - Add env var for supervisor.conf path

awx/main/tasks.py

@@ -141,6 +141,9 @@ def dispatch_startup():
     # and Tower fall out of use/support, we can probably just _assume_ that
     # everybody has moved to bigint, and remove this code entirely
     enforce_bigint_pk_migration()
+    
+    # Update Tower's rsyslog.conf file based on loggins settings in the db
+    reconfigure_rsyslog()
 
 
 def inform_cluster_of_shutdown():

awx/main/utils/external_logging.py

@@ -7,6 +7,7 @@ from awx.main.utils.reload import supervisor_service_command
 
 def reconfigure_rsyslog():
     tmpl = ''
+    parts = ['$IncludeConfig /etc/rsyslog.conf']
     if settings.LOG_AGGREGATOR_ENABLED:
         host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
         port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
@@ -26,11 +27,8 @@ def reconfigure_rsyslog():
             except ValueError:
                 port = settings.LOG_AGGREGATOR_PORT
 
-        parts = []
         parts.extend([
-            '$IncludeConfig /etc/rsyslog.conf',
+            'input(type="imuxsock" Socket="/var/run/tower/sockets/rsyslog.sock" unlink="on")',
-            '$ModLoad imudp',
-            '$UDPServerRun 51414',
             'template(name="awx" type="string" string="%msg%")',
         ])
         if protocol.startswith('http'):
@@ -65,8 +63,8 @@ def reconfigure_rsyslog():
             parts.append(
                 f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
             )
-        tmpl = '\n'.join(parts)
 
-    with open('/var/lib/awx/rsyslog.conf', 'w') as f:
+    tmpl = '\n'.join(parts)
+    with open('/var/lib/awx/rsyslog/rsyslog.conf', 'w') as f:
         f.write(tmpl + '\n')
     supervisor_service_command(command='restart', service='awx-rsyslogd')

awx/main/utils/handlers.py

@@ -3,10 +3,20 @@
 
 # Python
 import logging
+import os.path
 
 # Django
 from django.conf import settings
 
+
+class RSysLogHandler(logging.handlers.SysLogHandler):
+
+    def emit(self, msg):
+        if not os.path.exists(settings.LOGGING_SOCK):
+            return
+        return super(RSysLogHandler, self).emit(msg)
+
+
 ColorHandler = logging.StreamHandler
 
 if settings.COLOR_LOGS is True:

awx/main/utils/reload.py

@@ -4,6 +4,7 @@
 # Python
 import subprocess
 import logging
+import os
 
 # Django
 from django.conf import settings
@@ -17,6 +18,11 @@ def supervisor_service_command(command, service='*', communicate=True):
     # supervisorctl restart tower-processes:receiver tower-processes:factcacher
     '''
     args = ['supervisorctl']
+
+    supervisor_config_path = os.getenv('SUPERVISOR_WEB_CONFIG_PATH', None)
+    if supervisor_config_path:
+        args.extend(['-c', supervisor_config_path])
+
     args.extend([command, ':'.join(['tower-processes', service])])
     logger.debug('Issuing command to {} services, args={}'.format(command, args))
     supervisor_process = subprocess.Popen(args, stdin=subprocess.PIPE,

awx/settings/defaults.py

@@ -955,6 +955,7 @@ CHANNEL_LAYERS = {
 }
 
 # Logging configuration.
+LOGGING_SOCK = '/var/run/tower/sockets/rsyslog.sock'
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -1011,9 +1012,9 @@ LOGGING = {
             'formatter': 'simple',
         },
         'external_logger': {
-            'class': 'logging.handlers.SysLogHandler',
+            'class': 'awx.main.utils.handlers.RSysLogHandler',
             'formatter': 'json',
-            'address': ('localhost', 51414),
+            'address': LOGGING_SOCK,
             'filters': ['external_log_enabled', 'dynamic_level_filter'],
         },
         'tower_warnings': {

installer/roles/image_build/files/supervisor.conf

@@ -46,8 +46,20 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
+[program:awx-rsyslogd]
+command = rsyslogd -n -i /var/run/tower/sockets/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
+autostart = true
+autorestart = true
+stopwaitsecs = 1
+stopsignal=KILL
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/stderr
+stdout_logfile_maxbytes=0
+
 [group:tower-processes]
-programs=nginx,uwsgi,daphne,wsbroadcast
+programs=nginx,uwsgi,daphne,wsbroadcast,awx-rsyslogd
 priority=5
 
 # TODO: Exit Handler
@@ -62,10 +74,10 @@ events=TICK_60
 priority=0
 
 [unix_http_server]
-file=/tmp/supervisor.sock
+file=/var/run/tower/sockets/supervisor.web.sock
 
 [supervisorctl]
-serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
+serverurl=unix:///var/run/tower/sockets/supervisor.web.sock ; use a unix:// URL  for a unix socket
 
 [rpcinterface:supervisor]
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

installer/roles/image_build/templates/Dockerfile.j2

@@ -99,10 +99,9 @@ RUN cd /usr/local/bin && \
 
 ADD rsyslog.repo /etc/yum.repos.d/
 RUN yum install -y rsyslog-omhttp
-RUN echo '$IncludeConfig /var/lib/awx/rsyslog.conf' >> /etc/rsyslog.conf
 
 # Pre-create things that we need to write to
-RUN for dir in /home/awx /var/log/tower /var/log/nginx /var/lib/nginx; \
+RUN for dir in /home/awx /var/run/supervisor /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /var/run/nginx.pid; \

installer/roles/kubernetes/templates/deployment.yml.j2

@@ -94,6 +94,10 @@ spec:
           ports:
             - containerPort: 8052
           volumeMounts:
+            - name: sockets
+              mountPath: "/var/run/tower/sockets/"
+            - name: rsyslog
+              mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
               mountPath: "/etc/pki/ca-trust/source/anchors/"
@@ -174,6 +178,10 @@ spec:
             - /usr/bin/launch_awx_task.sh
           imagePullPolicy: Always
           volumeMounts:
+            - name: sockets
+              mountPath: "/var/run/tower/sockets/"
+            - name: rsyslog
+              mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
               mountPath: "/etc/pki/ca-trust/source/anchors/"
@@ -223,6 +231,8 @@ spec:
             - name: {{ kubernetes_deployment_name }}-memcached-socket
               mountPath: "/var/run/memcached"
           env:
+            - name: SUPERVISOR_WEB_CONFIG_PATH
+              value: "/supervisor.conf"
             - name: AWX_SKIP_MIGRATIONS
               value: "1"
             - name: MY_POD_UID
@@ -313,6 +323,10 @@ spec:
 {{ affinity | to_nice_yaml(indent=2) | indent(width=8, indentfirst=True) }}
 {% endif %}
       volumes:
+        - name: sockets
+          emptyDir: {}
+        - name: rsyslog
+          emptyDir: {}
 {% if ca_trust_dir is defined %}
         - name: {{ kubernetes_deployment_name }}-ca-trust-dir
           hostPath:

installer/roles/local_docker/templates/docker-compose.yml.j2

@@ -20,6 +20,8 @@ services:
     user: root
     restart: unless-stopped
     volumes:
+      - sockets:/var/run/tower/sockets/
+      - rsyslog:/var/lib/awx/rsyslog/
       - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY"
       - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh"
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
@@ -75,6 +77,8 @@ services:
     user: root
     restart: unless-stopped
     volumes:
+      - sockets:/var/run/tower/sockets/
+      - rsyslog:/var/lib/awx/rsyslog/
       - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY"
       - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh"
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
@@ -114,6 +118,7 @@ services:
       http_proxy: {{ http_proxy | default('') }}
       https_proxy: {{ https_proxy | default('') }}
       no_proxy: {{ no_proxy | default('') }}
+      SUPERVISOR_WEB_CONFIG_PATH: '/supervisor.conf'
 
   redis:
     image: {{ redis_image }}
@@ -157,3 +162,6 @@ services:
       https_proxy: {{ https_proxy | default('') }}
       no_proxy: {{ no_proxy | default('') }}
   {% endif %}
+volumes:
+  sockets:
+  rsyslog:

tools/docker-compose/Dockerfile

@@ -104,7 +104,7 @@ RUN cd /usr/local/bin && \
 
 ADD tools/docker-compose/rsyslog.repo /etc/yum.repos.d/
 RUN yum install -y rsyslog-omhttp
-RUN echo '$IncludeConfig /var/lib/awx/rsyslog.conf' >> /etc/rsyslog.conf
+RUN mkdir -p /var/lib/awx/rsyslog/ && echo '$IncludeConfig /etc/rsyslog.conf' >> /var/lib/awx/rsyslog/rsyslog.conf
 
 RUN dnf -y clean all && rm -rf /root/.cache
 
@@ -123,11 +123,20 @@ ADD tools/docker-compose/entrypoint.sh /
 ADD tools/scripts/awx-python /usr/bin/awx-python
 
 # Pre-create things that we need to write to
+<<<<<<< HEAD
 RUN for dir in /var/lib/awx/ /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \
     do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done
+=======
+RUN for dir in /var/lib/awx/rsyslog /var/run/tower/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
+  do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done
+
+RUN for file in /etc/passwd /etc/supervisord.conf \
+  /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \
+  do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done
+>>>>>>> Configure rsyslog to listen over a unix domain socket instead of a port
 
 ENV HOME /var/lib/awx
 ENV PATH="/usr/local/n/versions/node/10.15.0/bin:${PATH}"

tools/docker-compose/supervisor.conf

@@ -72,7 +72,7 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
 [program:awx-rsyslogd]
-command = rsyslogd -n -i /awx_devel/rsyslog.pid
+command = rsyslogd -n -i /var/run/tower/rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
 autostart = true
 autorestart = true
 stopwaitsecs = 1