From c0af3c537be66c19b87e4617ee170c302c0d7f17 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 25 Feb 2020 19:55:14 -0500 Subject: [PATCH] Configure rsyslog to listen over a unix domain socket instead of a port - Add a placeholder rsyslog.conf so it doesn't fail on start - Create access restricted directory for unix socket to be created in - Create RSyslogHandler to exit early when logging socket doesn't exist - Write updated logging settings when dispatcher comes up and restart rsyslog so they take effect - Move rsyslogd to the web container and create rpc supervisor.sock - Add env var for supervisor.conf path --- awx/main/tasks.py | 3 +++ awx/main/utils/external_logging.py | 10 ++++------ awx/main/utils/handlers.py | 10 ++++++++++ awx/main/utils/reload.py | 6 ++++++ awx/settings/defaults.py | 5 +++-- .../roles/image_build/files/supervisor.conf | 18 +++++++++++++++--- .../roles/image_build/templates/Dockerfile.j2 | 3 +-- .../kubernetes/templates/deployment.yml.j2 | 14 ++++++++++++++ .../templates/docker-compose.yml.j2 | 8 ++++++++ tools/docker-compose/Dockerfile | 11 ++++++++++- tools/docker-compose/supervisor.conf | 2 +- 11 files changed, 75 insertions(+), 15 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 3116351644..ebf29bbe97 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -141,6 +141,9 @@ def dispatch_startup(): # and Tower fall out of use/support, we can probably just _assume_ that # everybody has moved to bigint, and remove this code entirely enforce_bigint_pk_migration() + + # Update Tower's rsyslog.conf file based on loggins settings in the db + reconfigure_rsyslog() def inform_cluster_of_shutdown(): diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py index bef6dac151..1d373293db 100644 --- a/awx/main/utils/external_logging.py +++ b/awx/main/utils/external_logging.py @@ -7,6 +7,7 @@ from awx.main.utils.reload import supervisor_service_command def reconfigure_rsyslog(): tmpl = '' + parts = ['$IncludeConfig /etc/rsyslog.conf'] if settings.LOG_AGGREGATOR_ENABLED: host = getattr(settings, 'LOG_AGGREGATOR_HOST', '') port = getattr(settings, 'LOG_AGGREGATOR_PORT', '') @@ -26,11 +27,8 @@ def reconfigure_rsyslog(): except ValueError: port = settings.LOG_AGGREGATOR_PORT - parts = [] parts.extend([ - '$IncludeConfig /etc/rsyslog.conf', - '$ModLoad imudp', - '$UDPServerRun 51414', + 'input(type="imuxsock" Socket="/var/run/tower/sockets/rsyslog.sock" unlink="on")', 'template(name="awx" type="string" string="%msg%")', ]) if protocol.startswith('http'): @@ -65,8 +63,8 @@ def reconfigure_rsyslog(): parts.append( f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")' # noqa ) - tmpl = '\n'.join(parts) - with open('/var/lib/awx/rsyslog.conf', 'w') as f: + tmpl = '\n'.join(parts) + with open('/var/lib/awx/rsyslog/rsyslog.conf', 'w') as f: f.write(tmpl + '\n') supervisor_service_command(command='restart', service='awx-rsyslogd') diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index 0640c14921..26b4282fd1 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -3,10 +3,20 @@ # Python import logging +import os.path # Django from django.conf import settings + +class RSysLogHandler(logging.handlers.SysLogHandler): + + def emit(self, msg): + if not os.path.exists(settings.LOGGING_SOCK): + return + return super(RSysLogHandler, self).emit(msg) + + ColorHandler = logging.StreamHandler if settings.COLOR_LOGS is True: diff --git a/awx/main/utils/reload.py b/awx/main/utils/reload.py index 7c8ff4c999..04868f06d0 100644 --- a/awx/main/utils/reload.py +++ b/awx/main/utils/reload.py @@ -4,6 +4,7 @@ # Python import subprocess import logging +import os # Django from django.conf import settings @@ -17,6 +18,11 @@ def supervisor_service_command(command, service='*', communicate=True): # supervisorctl restart tower-processes:receiver tower-processes:factcacher ''' args = ['supervisorctl'] + + supervisor_config_path = os.getenv('SUPERVISOR_WEB_CONFIG_PATH', None) + if supervisor_config_path: + args.extend(['-c', supervisor_config_path]) + args.extend([command, ':'.join(['tower-processes', service])]) logger.debug('Issuing command to {} services, args={}'.format(command, args)) supervisor_process = subprocess.Popen(args, stdin=subprocess.PIPE, diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index a53e65bfad..0207f92b55 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -955,6 +955,7 @@ CHANNEL_LAYERS = { } # Logging configuration. +LOGGING_SOCK = '/var/run/tower/sockets/rsyslog.sock' LOGGING = { 'version': 1, 'disable_existing_loggers': False, @@ -1011,9 +1012,9 @@ LOGGING = { 'formatter': 'simple', }, 'external_logger': { - 'class': 'logging.handlers.SysLogHandler', + 'class': 'awx.main.utils.handlers.RSysLogHandler', 'formatter': 'json', - 'address': ('localhost', 51414), + 'address': LOGGING_SOCK, 'filters': ['external_log_enabled', 'dynamic_level_filter'], }, 'tower_warnings': { diff --git a/installer/roles/image_build/files/supervisor.conf b/installer/roles/image_build/files/supervisor.conf index acc1af1d6b..bd49186170 100644 --- a/installer/roles/image_build/files/supervisor.conf +++ b/installer/roles/image_build/files/supervisor.conf @@ -46,8 +46,20 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 +[program:awx-rsyslogd] +command = rsyslogd -n -i /var/run/tower/sockets/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf +autostart = true +autorestart = true +stopwaitsecs = 1 +stopsignal=KILL +stopasgroup=true +killasgroup=true +redirect_stderr=true +stdout_logfile=/dev/stderr +stdout_logfile_maxbytes=0 + [group:tower-processes] -programs=nginx,uwsgi,daphne,wsbroadcast +programs=nginx,uwsgi,daphne,wsbroadcast,awx-rsyslogd priority=5 # TODO: Exit Handler @@ -62,10 +74,10 @@ events=TICK_60 priority=0 [unix_http_server] -file=/tmp/supervisor.sock +file=/var/run/tower/sockets/supervisor.web.sock [supervisorctl] -serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket +serverurl=unix:///var/run/tower/sockets/supervisor.web.sock ; use a unix:// URL for a unix socket [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2 index 44668ec6b8..8b829e7976 100644 --- a/installer/roles/image_build/templates/Dockerfile.j2 +++ b/installer/roles/image_build/templates/Dockerfile.j2 @@ -99,10 +99,9 @@ RUN cd /usr/local/bin && \ ADD rsyslog.repo /etc/yum.repos.d/ RUN yum install -y rsyslog-omhttp -RUN echo '$IncludeConfig /var/lib/awx/rsyslog.conf' >> /etc/rsyslog.conf # Pre-create things that we need to write to -RUN for dir in /home/awx /var/log/tower /var/log/nginx /var/lib/nginx; \ +RUN for dir in /home/awx /var/run/supervisor /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \ do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \ \ for file in /etc/passwd /var/run/nginx.pid; \ diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index 869fbb0ffb..cdf414f28b 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -94,6 +94,10 @@ spec: ports: - containerPort: 8052 volumeMounts: + - name: sockets + mountPath: "/var/run/tower/sockets/" + - name: rsyslog + mountPath: "/var/lib/awx/rsyslog/" {% if ca_trust_dir is defined %} - name: {{ kubernetes_deployment_name }}-ca-trust-dir mountPath: "/etc/pki/ca-trust/source/anchors/" @@ -174,6 +178,10 @@ spec: - /usr/bin/launch_awx_task.sh imagePullPolicy: Always volumeMounts: + - name: sockets + mountPath: "/var/run/tower/sockets/" + - name: rsyslog + mountPath: "/var/lib/awx/rsyslog/" {% if ca_trust_dir is defined %} - name: {{ kubernetes_deployment_name }}-ca-trust-dir mountPath: "/etc/pki/ca-trust/source/anchors/" @@ -223,6 +231,8 @@ spec: - name: {{ kubernetes_deployment_name }}-memcached-socket mountPath: "/var/run/memcached" env: + - name: SUPERVISOR_WEB_CONFIG_PATH + value: "/supervisor.conf" - name: AWX_SKIP_MIGRATIONS value: "1" - name: MY_POD_UID @@ -313,6 +323,10 @@ spec: {{ affinity | to_nice_yaml(indent=2) | indent(width=8, indentfirst=True) }} {% endif %} volumes: + - name: sockets + emptyDir: {} + - name: rsyslog + emptyDir: {} {% if ca_trust_dir is defined %} - name: {{ kubernetes_deployment_name }}-ca-trust-dir hostPath: diff --git a/installer/roles/local_docker/templates/docker-compose.yml.j2 b/installer/roles/local_docker/templates/docker-compose.yml.j2 index e9a26f4416..3dfb449a73 100644 --- a/installer/roles/local_docker/templates/docker-compose.yml.j2 +++ b/installer/roles/local_docker/templates/docker-compose.yml.j2 @@ -20,6 +20,8 @@ services: user: root restart: unless-stopped volumes: + - sockets:/var/run/tower/sockets/ + - rsyslog:/var/lib/awx/rsyslog/ - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY" - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh" - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py" @@ -75,6 +77,8 @@ services: user: root restart: unless-stopped volumes: + - sockets:/var/run/tower/sockets/ + - rsyslog:/var/lib/awx/rsyslog/ - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY" - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh" - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py" @@ -114,6 +118,7 @@ services: http_proxy: {{ http_proxy | default('') }} https_proxy: {{ https_proxy | default('') }} no_proxy: {{ no_proxy | default('') }} + SUPERVISOR_WEB_CONFIG_PATH: '/supervisor.conf' redis: image: {{ redis_image }} @@ -157,3 +162,6 @@ services: https_proxy: {{ https_proxy | default('') }} no_proxy: {{ no_proxy | default('') }} {% endif %} +volumes: + sockets: + rsyslog: diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile index f59f466a6f..7fd9b8e74b 100644 --- a/tools/docker-compose/Dockerfile +++ b/tools/docker-compose/Dockerfile @@ -104,7 +104,7 @@ RUN cd /usr/local/bin && \ ADD tools/docker-compose/rsyslog.repo /etc/yum.repos.d/ RUN yum install -y rsyslog-omhttp -RUN echo '$IncludeConfig /var/lib/awx/rsyslog.conf' >> /etc/rsyslog.conf +RUN mkdir -p /var/lib/awx/rsyslog/ && echo '$IncludeConfig /etc/rsyslog.conf' >> /var/lib/awx/rsyslog/rsyslog.conf RUN dnf -y clean all && rm -rf /root/.cache @@ -123,11 +123,20 @@ ADD tools/docker-compose/entrypoint.sh / ADD tools/scripts/awx-python /usr/bin/awx-python # Pre-create things that we need to write to +<<<<<<< HEAD RUN for dir in /var/lib/awx/ /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \ do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \ \ for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \ do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done +======= +RUN for dir in /var/lib/awx/rsyslog /var/run/tower/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \ + do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done + +RUN for file in /etc/passwd /etc/supervisord.conf \ + /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \ + do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done +>>>>>>> Configure rsyslog to listen over a unix domain socket instead of a port ENV HOME /var/lib/awx ENV PATH="/usr/local/n/versions/node/10.15.0/bin:${PATH}" diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf index f5416c4b26..826088c6ad 100644 --- a/tools/docker-compose/supervisor.conf +++ b/tools/docker-compose/supervisor.conf @@ -72,7 +72,7 @@ stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 [program:awx-rsyslogd] -command = rsyslogd -n -i /awx_devel/rsyslog.pid +command = rsyslogd -n -i /var/run/tower/rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf autostart = true autorestart = true stopwaitsecs = 1