POC: replace our external log aggregation feature with rsyslog

- this change adds rsyslog (https://github.com/rsyslog/rsyslog) as
  a new service that runs on every AWX node (managed by supervisord)
  in particular, this feature requires a recent version (v8.38+) of
  rsyslog that supports the omhttp module
  (https://github.com/rsyslog/rsyslog-doc/pull/750)
- the "external_logger" handler in AWX is now a SysLogHandler that ships
  logs to the local UDP port where rsyslog is configured to listen (by
  default, 51414)
- every time a LOG_AGGREGATOR_* setting is changed, every AWX node
  reconfigures and restarts its local instance of rsyslog so that its
  fowarding settings match what has been configured in AWX
- unlike the prior implementation, if the external logging aggregator
  (splunk/logstash) goes temporarily offline, rsyslog will retain the
  messages and ship them when the log aggregator is back online
- 4xx or 5xx level errors are recorded at /var/log/tower/external.err

tools/docker-compose/Dockerfile

@@ -102,6 +102,10 @@ RUN cd /usr/local/bin && \
     curl -L https://github.com/openshift/origin/releases/download/v3.9.0/openshift-origin-client-tools-v3.9.0-191fece-linux-64bit.tar.gz | \
     tar -xz --strip-components=1 --wildcards --no-anchored 'oc'
 
+ADD tools/docker-compose/rsyslog.repo /etc/yum.repos.d/
+RUN yum install -y rsyslog-omhttp
+RUN echo '$IncludeConfig /var/lib/awx/rsyslog.conf' >> /etc/rsyslog.conf
+
 RUN dnf -y clean all && rm -rf /root/.cache
 
 # https://github.com/ansible/awx/issues/5224

tools/docker-compose/rsyslog.repo

@@ -0,0 +1,7 @@
+[rsyslog_v8]
+name=Adiscon CentOS-$releasever - local packages for $basearch
+baseurl=http://rpms.adiscon.com/v8-stable/epel-$releasever/$basearch
+enabled=1
+gpgcheck=0
+gpgkey=http://rpms.adiscon.com/RPM-GPG-KEY-Adiscon
+protect=1

tools/docker-compose/supervisor.conf

@@ -71,8 +71,20 @@ redirect_stderr=true
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
+[program:awx-rsyslogd]
+command = rsyslogd -nd -i /awx_devel/rsyslog.pid
+autostart = true
+autorestart = true
+stopwaitsecs = 1
+stopsignal=KILL
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/tmp/ryan
+stdout_logfile_maxbytes=0
+
 [group:tower-processes]
-programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast
+programs=awx-dispatcher,awx-receiver,awx-runworker,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast,awx-rsyslogd
 priority=5
 
 [unix_http_server]