From 589d27c88c4d514026c1425f4f7fa8cda188156e Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Wed, 23 Oct 2019 23:54:47 -0400
Subject: [PATCH 01/37] POC: replace our external log aggregation feature with
 rsyslog

- this change adds rsyslog (https://github.com/rsyslog/rsyslog) as
  a new service that runs on every AWX node (managed by supervisord)
  in particular, this feature requires a recent version (v8.38+) of
  rsyslog that supports the omhttp module
  (https://github.com/rsyslog/rsyslog-doc/pull/750)
- the "external_logger" handler in AWX is now a SysLogHandler that ships
  logs to the local UDP port where rsyslog is configured to listen (by
  default, 51414)
- every time a LOG_AGGREGATOR_* setting is changed, every AWX node
  reconfigures and restarts its local instance of rsyslog so that its
  fowarding settings match what has been configured in AWX
- unlike the prior implementation, if the external logging aggregator
  (splunk/logstash) goes temporarily offline, rsyslog will retain the
  messages and ship them when the log aggregator is back online
- 4xx or 5xx level errors are recorded at /var/log/tower/external.err
---
 awx/conf/views.py                             |  38 +-
 .../management/commands/run_dispatcher.py     |   6 -
 awx/main/tasks.py                             |   9 +-
 awx/main/tests/unit/utils/test_handlers.py    | 393 -----------------
 awx/main/utils/external_logging.py            |  65 +++
 awx/main/utils/handlers.py                    | 396 ------------------
 awx/main/utils/reload.py                      |  11 +-
 awx/settings/defaults.py                      |   3 +-
 .../sub-forms/system-logging.form.js          |   2 +-
 docs/licenses/requests-futures.txt            |  13 -
 requirements/requirements.in                  |   1 -
 tools/docker-compose/Dockerfile               |   4 +
 tools/docker-compose/rsyslog.repo             |   7 +
 tools/docker-compose/supervisor.conf          |  14 +-
 14 files changed, 106 insertions(+), 856 deletions(-)
 delete mode 100644 awx/main/tests/unit/utils/test_handlers.py
 create mode 100644 awx/main/utils/external_logging.py
 delete mode 100644 docs/licenses/requests-futures.txt
 create mode 100644 tools/docker-compose/rsyslog.repo

diff --git a/awx/conf/views.py b/awx/conf/views.py
index 13b72a926f..7a2a21a713 100644
--- a/awx/conf/views.py
+++ b/awx/conf/views.py
@@ -3,6 +3,7 @@
 
 # Python
 import collections
+import logging
 import sys
 
 # Django
@@ -26,7 +27,6 @@ from awx.api.generics import (
 from awx.api.permissions import IsSuperUser
 from awx.api.versioning import reverse
 from awx.main.utils import camelcase_to_underscore
-from awx.main.utils.handlers import AWXProxyHandler, LoggingConnectivityException
 from awx.main.tasks import handle_setting_changes
 from awx.conf.models import Setting
 from awx.conf.serializers import SettingCategorySerializer, SettingSingletonSerializer
@@ -161,40 +161,8 @@ class SettingLoggingTest(GenericAPIView):
     filter_backends = []
 
     def post(self, request, *args, **kwargs):
-        defaults = dict()
-        for key in settings_registry.get_registered_settings(category_slug='logging'):
-            try:
-                defaults[key] = settings_registry.get_setting_field(key).get_default()
-            except serializers.SkipField:
-                defaults[key] = None
-        obj = type('Settings', (object,), defaults)()
-        serializer = self.get_serializer(obj, data=request.data)
-        serializer.is_valid(raise_exception=True)
-        # Special validation specific to logging test.
-        errors = {}
-        for key in ['LOG_AGGREGATOR_TYPE', 'LOG_AGGREGATOR_HOST']:
-            if not request.data.get(key, ''):
-                errors[key] = 'This field is required.'
-        if errors:
-            raise ValidationError(errors)
-
-        if request.data.get('LOG_AGGREGATOR_PASSWORD', '').startswith('$encrypted$'):
-            serializer.validated_data['LOG_AGGREGATOR_PASSWORD'] = getattr(
-                settings, 'LOG_AGGREGATOR_PASSWORD', ''
-            )
-
-        try:
-            class MockSettings:
-                pass
-            mock_settings = MockSettings()
-            for k, v in serializer.validated_data.items():
-                setattr(mock_settings, k, v)
-            AWXProxyHandler().perform_test(custom_settings=mock_settings)
-            if mock_settings.LOG_AGGREGATOR_PROTOCOL.upper() == 'UDP':
-                return Response(status=status.HTTP_201_CREATED)
-        except LoggingConnectivityException as e:
-            return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-        return Response(status=status.HTTP_200_OK)
+        logging.getLogger('awx').info('AWX Connection Test')
+        return Response(status=status.HTTP_202_ACCEPTED)
 
 
 # Create view functions for all of the class-based views to simplify inclusion
diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py
index 5f7db4f106..e22baa379f 100644
--- a/awx/main/management/commands/run_dispatcher.py
+++ b/awx/main/management/commands/run_dispatcher.py
@@ -7,7 +7,6 @@ from django.core.cache import cache as django_cache
 from django.core.management.base import BaseCommand
 from django.db import connection as django_connection
 
-from awx.main.utils.handlers import AWXProxyHandler
 from awx.main.dispatch import get_local_queuename, reaper
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.pool import AutoscalePool
@@ -56,11 +55,6 @@ class Command(BaseCommand):
         reaper.reap()
         consumer = None
 
-        # don't ship external logs inside the dispatcher's parent process
-        # this exists to work around a race condition + deadlock bug on fork
-        # in cpython itself:
-        # https://bugs.python.org/issue37429
-        AWXProxyHandler.disable()
         try:
             queues = ['tower_broadcast_all', get_local_queuename()]
             consumer = AWXConsumerPG(
diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index 6374815769..3116351644 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -72,7 +72,8 @@ from awx.main.utils import (get_ssh_version, update_scm_url,
                             ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager,
                             get_awx_version)
 from awx.main.utils.ansible import read_ansible_config
-from awx.main.utils.common import _get_ansible_version, get_custom_venv_choices
+from awx.main.utils.common import get_ansible_version, _get_ansible_version, get_custom_venv_choices
+from awx.main.utils.external_logging import reconfigure_rsyslog
 from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja
 from awx.main.utils.reload import stop_local_services
 from awx.main.utils.pglock import advisory_lock
@@ -280,6 +281,12 @@ def handle_setting_changes(setting_keys):
     logger.debug('cache delete_many(%r)', cache_keys)
     cache.delete_many(cache_keys)
 
+    if any([
+        setting.startswith('LOG_AGGREGATOR')
+        for setting in setting_keys
+    ]):
+        reconfigure_rsyslog()
+
 
 @task(queue='tower_broadcast_all')
 def delete_project_files(project_path):
diff --git a/awx/main/tests/unit/utils/test_handlers.py b/awx/main/tests/unit/utils/test_handlers.py
deleted file mode 100644
index 6fa9b1f992..0000000000
--- a/awx/main/tests/unit/utils/test_handlers.py
+++ /dev/null
@@ -1,393 +0,0 @@
-# -*- coding: utf-8 -*-
-import base64
-import logging
-import socket
-import datetime
-from dateutil.tz import tzutc
-from io import StringIO
-from uuid import uuid4
-
-from unittest import mock
-
-from django.conf import LazySettings
-from django.utils.encoding import smart_str
-import pytest
-import requests
-from requests_futures.sessions import FuturesSession
-
-from awx.main.utils.handlers import (BaseHandler, BaseHTTPSHandler as HTTPSHandler,
-                                     TCPHandler, UDPHandler, _encode_payload_for_socket,
-                                     PARAM_NAMES, LoggingConnectivityException,
-                                     AWXProxyHandler)
-from awx.main.utils.formatters import LogstashFormatter
-
-
-@pytest.fixture()
-def https_adapter():
-    class FakeHTTPSAdapter(requests.adapters.HTTPAdapter):
-        requests = []
-        status = 200
-        reason = None
-
-        def send(self, request, **kwargs):
-            self.requests.append(request)
-            resp = requests.models.Response()
-            resp.status_code = self.status
-            resp.reason = self.reason
-            resp.request = request
-            return resp
-
-    return FakeHTTPSAdapter()
-
-
-@pytest.fixture()
-def connection_error_adapter():
-    class ConnectionErrorAdapter(requests.adapters.HTTPAdapter):
-
-        def send(self, request, **kwargs):
-            err = requests.packages.urllib3.exceptions.SSLError()
-            raise requests.exceptions.ConnectionError(err, request=request)
-
-    return ConnectionErrorAdapter()
-
-
-@pytest.fixture
-def fake_socket(tmpdir_factory, request):
-    sok = socket.socket
-    sok.send = mock.MagicMock()
-    sok.connect = mock.MagicMock()
-    sok.setblocking = mock.MagicMock()
-    sok.close = mock.MagicMock()
-    return sok
-
-
-def test_https_logging_handler_requests_async_implementation():
-    handler = HTTPSHandler()
-    assert isinstance(handler.session, FuturesSession)
-
-
-def test_https_logging_handler_has_default_http_timeout():
-    handler = TCPHandler()
-    assert handler.tcp_timeout == 5
-
-
-@pytest.mark.parametrize('param', ['host', 'port', 'indv_facts'])
-def test_base_logging_handler_defaults(param):
-    handler = BaseHandler()
-    assert hasattr(handler, param) and getattr(handler, param) is None
-
-
-@pytest.mark.parametrize('param', ['host', 'port', 'indv_facts'])
-def test_base_logging_handler_kwargs(param):
-    handler = BaseHandler(**{param: 'EXAMPLE'})
-    assert hasattr(handler, param) and getattr(handler, param) == 'EXAMPLE'
-
-
-@pytest.mark.parametrize('params', [
-    {
-        'LOG_AGGREGATOR_HOST': 'https://server.invalid',
-        'LOG_AGGREGATOR_PORT': 22222,
-        'LOG_AGGREGATOR_TYPE': 'loggly',
-        'LOG_AGGREGATOR_USERNAME': 'foo',
-        'LOG_AGGREGATOR_PASSWORD': 'bar',
-        'LOG_AGGREGATOR_INDIVIDUAL_FACTS': True,
-        'LOG_AGGREGATOR_TCP_TIMEOUT': 96,
-        'LOG_AGGREGATOR_VERIFY_CERT': False,
-        'LOG_AGGREGATOR_PROTOCOL': 'https'
-    },
-    {
-        'LOG_AGGREGATOR_HOST': 'https://server.invalid',
-        'LOG_AGGREGATOR_PORT': 22222,
-        'LOG_AGGREGATOR_PROTOCOL': 'udp'
-    }
-])
-def test_real_handler_from_django_settings(params):
-    settings = LazySettings()
-    settings.configure(**params)
-    handler = AWXProxyHandler().get_handler(custom_settings=settings)
-    # need the _reverse_ dictionary from PARAM_NAMES
-    attr_lookup = {}
-    for attr_name, setting_name in PARAM_NAMES.items():
-        attr_lookup[setting_name] = attr_name
-    for setting_name, val in params.items():
-        attr_name = attr_lookup[setting_name]
-        if attr_name == 'protocol':
-            continue
-        assert hasattr(handler, attr_name)
-
-
-def test_invalid_kwarg_to_real_handler():
-    settings = LazySettings()
-    settings.configure(**{
-        'LOG_AGGREGATOR_HOST': 'https://server.invalid',
-        'LOG_AGGREGATOR_PORT': 22222,
-        'LOG_AGGREGATOR_PROTOCOL': 'udp',
-        'LOG_AGGREGATOR_VERIFY_CERT': False  # setting not valid for UDP handler
-    })
-    handler = AWXProxyHandler().get_handler(custom_settings=settings)
-    assert not hasattr(handler, 'verify_cert')
-
-
-def test_protocol_not_specified():
-    settings = LazySettings()
-    settings.configure(**{
-        'LOG_AGGREGATOR_HOST': 'https://server.invalid',
-        'LOG_AGGREGATOR_PORT': 22222,
-        'LOG_AGGREGATOR_PROTOCOL': None  # awx/settings/defaults.py
-    })
-    handler = AWXProxyHandler().get_handler(custom_settings=settings)
-    assert isinstance(handler, logging.NullHandler)
-
-
-def test_base_logging_handler_emit_system_tracking(dummy_log_record):
-    handler = BaseHandler(host='127.0.0.1', indv_facts=True)
-    handler.setFormatter(LogstashFormatter())
-    dummy_log_record.name = 'awx.analytics.system_tracking'
-    dummy_log_record.msg = None
-    dummy_log_record.inventory_id = 11
-    dummy_log_record.host_name = 'my_lucky_host'
-    dummy_log_record.job_id = 777
-    dummy_log_record.ansible_facts = {
-        "ansible_kernel": "4.4.66-boot2docker",
-        "ansible_machine": "x86_64",
-        "ansible_swapfree_mb": 4663,
-    }
-    dummy_log_record.ansible_facts_modified = datetime.datetime.now(tzutc()).isoformat()
-    sent_payloads = handler.emit(dummy_log_record)
-
-    assert len(sent_payloads) == 1
-    assert sent_payloads[0]['ansible_facts'] == dummy_log_record.ansible_facts
-    assert sent_payloads[0]['ansible_facts_modified'] == dummy_log_record.ansible_facts_modified
-    assert sent_payloads[0]['level'] == 'INFO'
-    assert sent_payloads[0]['logger_name'] == 'awx.analytics.system_tracking'
-    assert sent_payloads[0]['job_id'] == dummy_log_record.job_id
-    assert sent_payloads[0]['inventory_id'] == dummy_log_record.inventory_id
-    assert sent_payloads[0]['host_name'] == dummy_log_record.host_name
-
-
-@pytest.mark.parametrize('host, port, normalized, hostname_only', [
-    ('http://localhost', None, 'http://localhost', False),
-    ('http://localhost', 8080, 'http://localhost:8080', False),
-    ('https://localhost', 443, 'https://localhost:443', False),
-    ('ftp://localhost', 443, 'ftp://localhost:443', False),
-    ('https://localhost:550', 443, 'https://localhost:550', False),
-    ('https://localhost:yoho/foobar', 443, 'https://localhost:443/foobar', False),
-    ('https://localhost:yoho/foobar', None, 'https://localhost:yoho/foobar', False),
-    ('http://splunk.server:8088/services/collector/event', 80,
-     'http://splunk.server:8088/services/collector/event', False),
-    ('http://splunk.server/services/collector/event', 8088,
-     'http://splunk.server:8088/services/collector/event', False),
-    ('splunk.server:8088/services/collector/event', 80,
-     'http://splunk.server:8088/services/collector/event', False),
-    ('splunk.server/services/collector/event', 8088,
-     'http://splunk.server:8088/services/collector/event', False),
-    ('localhost', None, 'http://localhost', False),
-    ('localhost', 8080, 'http://localhost:8080', False),
-    ('localhost', 4399, 'localhost', True),
-    ('tcp://localhost:4399/foo/bar', 4399, 'localhost', True),
-])
-def test_base_logging_handler_host_format(host, port, normalized, hostname_only):
-    handler = BaseHandler(host=host, port=port)
-    assert handler._get_host(scheme='http', hostname_only=hostname_only) == normalized
-
-
-@pytest.mark.parametrize(
-    'status, reason, exc',
-    [(200, '200 OK', None), (404, 'Not Found', LoggingConnectivityException)]
-)
-@pytest.mark.parametrize('protocol', ['http', 'https', None])
-def test_https_logging_handler_connectivity_test(https_adapter, status, reason, exc, protocol):
-    host = 'example.org'
-    if protocol:
-        host = '://'.join([protocol, host])
-    https_adapter.status = status
-    https_adapter.reason = reason
-    settings = LazySettings()
-    settings.configure(**{
-        'LOG_AGGREGATOR_HOST': host,
-        'LOG_AGGREGATOR_PORT': 8080,
-        'LOG_AGGREGATOR_TYPE': 'logstash',
-        'LOG_AGGREGATOR_USERNAME': 'user',
-        'LOG_AGGREGATOR_PASSWORD': 'password',
-        'LOG_AGGREGATOR_LOGGERS': ['awx', 'activity_stream', 'job_events', 'system_tracking'],
-        'LOG_AGGREGATOR_PROTOCOL': 'https',
-        'CLUSTER_HOST_ID': '',
-        'LOG_AGGREGATOR_TOWER_UUID': str(uuid4()),
-        'LOG_AGGREGATOR_LEVEL': 'DEBUG',
-    })
-
-    class FakeHTTPSHandler(HTTPSHandler):
-
-        def __init__(self, *args, **kwargs):
-            super(FakeHTTPSHandler, self).__init__(*args, **kwargs)
-            self.session.mount('{}://'.format(protocol or 'https'), https_adapter)
-
-        def emit(self, record):
-            return super(FakeHTTPSHandler, self).emit(record)
-
-    with mock.patch.object(AWXProxyHandler, 'get_handler_class') as mock_get_class:
-        mock_get_class.return_value = FakeHTTPSHandler
-        if exc:
-            with pytest.raises(exc) as e:
-                AWXProxyHandler().perform_test(settings)
-            assert str(e).endswith('%s: %s' % (status, reason))
-        else:
-            assert AWXProxyHandler().perform_test(settings) is None
-
-
-def test_https_logging_handler_logstash_auth_info():
-    handler = HTTPSHandler(message_type='logstash', username='bob', password='ansible')
-    handler._add_auth_information()
-    assert isinstance(handler.session.auth, requests.auth.HTTPBasicAuth)
-    assert handler.session.auth.username == 'bob'
-    assert handler.session.auth.password == 'ansible'
-
-
-def test_https_logging_handler_splunk_auth_info():
-    handler = HTTPSHandler(message_type='splunk', password='ansible')
-    handler._add_auth_information()
-    assert handler.session.headers['Authorization'] == 'Splunk ansible'
-    assert handler.session.headers['Content-Type'] == 'application/json'
-
-
-def test_https_logging_handler_connection_error(connection_error_adapter,
-                                                dummy_log_record):
-    handler = HTTPSHandler(host='127.0.0.1', message_type='logstash')
-    handler.setFormatter(LogstashFormatter())
-    handler.session.mount('http://', connection_error_adapter)
-
-    buff = StringIO()
-    logging.getLogger('awx.main.utils.handlers').addHandler(
-        logging.StreamHandler(buff)
-    )
-
-    async_futures = handler.emit(dummy_log_record)
-    with pytest.raises(requests.exceptions.ConnectionError):
-        [future.result() for future in async_futures]
-    assert 'failed to emit log to external aggregator\nTraceback' in buff.getvalue()
-
-    # we should only log failures *periodically*, so causing *another*
-    # immediate failure shouldn't report a second ConnectionError
-    buff.truncate(0)
-    async_futures = handler.emit(dummy_log_record)
-    with pytest.raises(requests.exceptions.ConnectionError):
-        [future.result() for future in async_futures]
-    assert buff.getvalue() == ''
-
-
-@pytest.mark.parametrize('message_type', ['logstash', 'splunk'])
-def test_https_logging_handler_emit_without_cred(https_adapter, dummy_log_record,
-                                                 message_type):
-    handler = HTTPSHandler(host='127.0.0.1', message_type=message_type)
-    handler.setFormatter(LogstashFormatter())
-    handler.session.mount('https://', https_adapter)
-    async_futures = handler.emit(dummy_log_record)
-    [future.result() for future in async_futures]
-
-    assert len(https_adapter.requests) == 1
-    request = https_adapter.requests[0]
-    assert request.url == 'https://127.0.0.1/'
-    assert request.method == 'POST'
-
-    if message_type == 'logstash':
-        # A username + password weren't used, so this header should be missing
-        assert 'Authorization' not in request.headers
-
-    if message_type == 'splunk':
-        assert request.headers['Authorization'] == 'Splunk None'
-
-
-def test_https_logging_handler_emit_logstash_with_creds(https_adapter,
-                                                        dummy_log_record):
-    handler = HTTPSHandler(host='127.0.0.1',
-                           username='user', password='pass',
-                           message_type='logstash')
-    handler.setFormatter(LogstashFormatter())
-    handler.session.mount('https://', https_adapter)
-    async_futures = handler.emit(dummy_log_record)
-    [future.result() for future in async_futures]
-
-    assert len(https_adapter.requests) == 1
-    request = https_adapter.requests[0]
-    assert request.headers['Authorization'] == 'Basic %s' % smart_str(base64.b64encode(b"user:pass"))
-
-
-def test_https_logging_handler_emit_splunk_with_creds(https_adapter,
-                                                      dummy_log_record):
-    handler = HTTPSHandler(host='127.0.0.1',
-                           password='pass', message_type='splunk')
-    handler.setFormatter(LogstashFormatter())
-    handler.session.mount('https://', https_adapter)
-    async_futures = handler.emit(dummy_log_record)
-    [future.result() for future in async_futures]
-
-    assert len(https_adapter.requests) == 1
-    request = https_adapter.requests[0]
-    assert request.headers['Authorization'] == 'Splunk pass'
-
-
-@pytest.mark.parametrize('payload, encoded_payload', [
-    ('foobar', 'foobar'),
-    ({'foo': 'bar'}, '{"foo": "bar"}'),
-    ({u'测试键': u'测试值'}, '{"测试键": "测试值"}'),
-])
-def test_encode_payload_for_socket(payload, encoded_payload):
-    assert _encode_payload_for_socket(payload).decode('utf-8') == encoded_payload
-
-
-def test_udp_handler_create_socket_at_init():
-    handler = UDPHandler(host='127.0.0.1', port=4399)
-    assert hasattr(handler, 'socket')
-    assert isinstance(handler.socket, socket.socket)
-    assert handler.socket.family == socket.AF_INET
-    assert handler.socket.type == socket.SOCK_DGRAM
-
-
-def test_udp_handler_send(dummy_log_record):
-    handler = UDPHandler(host='127.0.0.1', port=4399)
-    handler.setFormatter(LogstashFormatter())
-    with mock.patch('awx.main.utils.handlers._encode_payload_for_socket', return_value="des") as encode_mock,\
-            mock.patch.object(handler, 'socket') as socket_mock:
-        handler.emit(dummy_log_record)
-        encode_mock.assert_called_once_with(handler.format(dummy_log_record))
-        socket_mock.sendto.assert_called_once_with("des", ('127.0.0.1', 4399))
-
-
-def test_tcp_handler_send(fake_socket, dummy_log_record):
-    handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5)
-    handler.setFormatter(LogstashFormatter())
-    with mock.patch('socket.socket', return_value=fake_socket) as sok_init_mock,\
-            mock.patch('select.select', return_value=([], [fake_socket], [])):
-        handler.emit(dummy_log_record)
-        sok_init_mock.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
-        fake_socket.connect.assert_called_once_with(('127.0.0.1', 4399))
-        fake_socket.setblocking.assert_called_once_with(0)
-        fake_socket.send.assert_called_once_with(handler.format(dummy_log_record))
-        fake_socket.close.assert_called_once()
-
-
-def test_tcp_handler_return_if_socket_unavailable(fake_socket, dummy_log_record):
-    handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5)
-    handler.setFormatter(LogstashFormatter())
-    with mock.patch('socket.socket', return_value=fake_socket) as sok_init_mock,\
-            mock.patch('select.select', return_value=([], [], [])):
-        handler.emit(dummy_log_record)
-        sok_init_mock.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
-        fake_socket.connect.assert_called_once_with(('127.0.0.1', 4399))
-        fake_socket.setblocking.assert_called_once_with(0)
-        assert not fake_socket.send.called
-        fake_socket.close.assert_called_once()
-
-
-def test_tcp_handler_log_exception(fake_socket, dummy_log_record):
-    handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5)
-    handler.setFormatter(LogstashFormatter())
-    with mock.patch('socket.socket', return_value=fake_socket) as sok_init_mock,\
-            mock.patch('select.select', return_value=([], [], [])),\
-            mock.patch('awx.main.utils.handlers.logger') as logger_mock:
-        fake_socket.connect.side_effect = Exception("foo")
-        handler.emit(dummy_log_record)
-        sok_init_mock.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM)
-        logger_mock.exception.assert_called_once()
-        fake_socket.close.assert_called_once()
-        assert not fake_socket.send.called
diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
new file mode 100644
index 0000000000..26c13b195b
--- /dev/null
+++ b/awx/main/utils/external_logging.py
@@ -0,0 +1,65 @@
+import urllib.parse as urlparse
+
+from django.conf import settings
+
+from awx.main.utils.reload import supervisor_service_command
+
+
+def reconfigure_rsyslog():
+    tmpl = ''
+    if settings.LOG_AGGREGATOR_ENABLED:
+        host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
+        port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
+        protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '')
+
+        if protocol.startswith('http'):
+            scheme = 'https'
+            # urlparse requires '//' to be provided if scheme is not specified
+            original_parsed = urlparse.urlsplit(host)
+            if (not original_parsed.scheme and not host.startswith('//')) or original_parsed.hostname is None:
+                host = '%s://%s' % (scheme, host) if scheme else '//%s' % host
+            parsed = urlparse.urlsplit(host)
+
+            host = parsed.hostname
+            try:
+                port = parsed.port or settings.LOG_AGGREGATOR_PORT
+            except ValueError:
+                port = settings.LOG_AGGREGATOR_PORT
+
+        parts = []
+        parts.extend([
+            '$ModLoad imudp',
+            '$UDPServerRun 51414',
+            'template(name="awx" type="string" string="%msg%")',
+        ])
+        if protocol.startswith('http'):
+            # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst
+            ssl = "on" if parsed.scheme == 'https' else "off"
+            skip_verify = "off" if settings.LOG_AGGREGATOR_VERIFY_CERT else "on"
+            params = [
+                'type="omhttp"',
+                f'server="{host}"',
+                f'serverport="{port}"',
+                f'usehttps="{ssl}"',
+                f'skipverifyhost="{skip_verify}"',
+                'action.resumeRetryCount="-1"',
+                'template="awx"',
+                'errorfile="/var/log/tower/external.err"',
+            ]
+            username = getattr(settings, 'LOG_AGGREGATOR_USERNAME', '')
+            password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '')
+            if username:
+                params.append(f'uid="{username}"')
+            if password:
+                params.append(f'pwd="{password}"')
+            params = ' '.join(params)
+            parts.extend(['module(load="omhttp")', f'action({params})'])
+        else:
+            parts.append(
+                f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
+            )
+        tmpl = '\n'.join(parts)
+
+    with open('/var/lib/awx/rsyslog.conf', 'w') as f:
+        f.write(tmpl + '\n')
+    supervisor_service_command(command='restart', service='awx-rsyslogd')
diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py
index 515d410eb9..0640c14921 100644
--- a/awx/main/utils/handlers.py
+++ b/awx/main/utils/handlers.py
@@ -3,406 +3,10 @@
 
 # Python
 import logging
-import json
-import os
-import requests
-import time
-import threading
-import socket
-import select
-from urllib import parse as urlparse
-from concurrent.futures import ThreadPoolExecutor
-from requests.exceptions import RequestException
 
 # Django
 from django.conf import settings
 
-# requests futures, a dependency used by these handlers
-from requests_futures.sessions import FuturesSession
-import cachetools
-
-# AWX
-from awx.main.utils.formatters import LogstashFormatter
-
-
-__all__ = ['BaseHTTPSHandler', 'TCPHandler', 'UDPHandler',
-           'AWXProxyHandler']
-
-
-logger = logging.getLogger('awx.main.utils.handlers')
-
-
-# Translation of parameter names to names in Django settings
-# logging settings category, only those related to handler / log emission
-PARAM_NAMES = {
-    'host': 'LOG_AGGREGATOR_HOST',
-    'port': 'LOG_AGGREGATOR_PORT',
-    'message_type': 'LOG_AGGREGATOR_TYPE',
-    'username': 'LOG_AGGREGATOR_USERNAME',
-    'password': 'LOG_AGGREGATOR_PASSWORD',
-    'indv_facts': 'LOG_AGGREGATOR_INDIVIDUAL_FACTS',
-    'tcp_timeout': 'LOG_AGGREGATOR_TCP_TIMEOUT',
-    'verify_cert': 'LOG_AGGREGATOR_VERIFY_CERT',
-    'protocol': 'LOG_AGGREGATOR_PROTOCOL'
-}
-
-
-def unused_callback(sess, resp):
-    pass
-
-
-class LoggingConnectivityException(Exception):
-    pass
-
-
-class VerboseThreadPoolExecutor(ThreadPoolExecutor):
-
-    last_log_emit = 0
-
-    def submit(self, func, *args, **kwargs):
-        def _wrapped(*args, **kwargs):
-            try:
-                return func(*args, **kwargs)
-            except Exception:
-                # If an exception occurs in a concurrent thread worker (like
-                # a ConnectionError or a read timeout), periodically log
-                # that failure.
-                #
-                # This approach isn't really thread-safe, so we could
-                # potentially log once per thread every 10 seconds, but it
-                # beats logging *every* failed HTTP request in a scenario where
-                # you've typo'd your log aggregator hostname.
-                now = time.time()
-                if now - self.last_log_emit > 10:
-                    logger.exception('failed to emit log to external aggregator')
-                    self.last_log_emit = now
-                raise
-        return super(VerboseThreadPoolExecutor, self).submit(_wrapped, *args,
-                                                             **kwargs)
-
-
-class SocketResult:
-    '''
-    A class to be the return type of methods that send data over a socket
-    allows object to be used in the same way as a request futures object
-    '''
-    def __init__(self, ok, reason=None):
-        self.ok = ok
-        self.reason = reason
-
-    def result(self):
-        return self
-
-
-class BaseHandler(logging.Handler):
-    def __init__(self, host=None, port=None, indv_facts=None, **kwargs):
-        super(BaseHandler, self).__init__()
-        self.host = host
-        self.port = port
-        self.indv_facts = indv_facts
-
-    def _send(self, payload):
-        """Actually send message to log aggregator.
-        """
-        return payload
-
-    def _format_and_send_record(self, record):
-        if self.indv_facts:
-            return [self._send(json.loads(self.format(record)))]
-        return [self._send(self.format(record))]
-
-    def emit(self, record):
-        """
-            Emit a log record.  Returns a list of zero or more
-            implementation-specific objects for tests.
-        """
-        try:
-            return self._format_and_send_record(record)
-        except (KeyboardInterrupt, SystemExit):
-            raise
-        except Exception:
-            self.handleError(record)
-
-    def _get_host(self, scheme='', hostname_only=False):
-        """Return the host name of log aggregator.
-        """
-        host = self.host or ''
-        # urlparse requires '//' to be provided if scheme is not specified
-        original_parsed = urlparse.urlsplit(host)
-        if (not original_parsed.scheme and not host.startswith('//')) or original_parsed.hostname is None:
-            host = '%s://%s' % (scheme, host) if scheme else '//%s' % host
-        parsed = urlparse.urlsplit(host)
-
-        if hostname_only:
-            return parsed.hostname
-
-        try:
-            port = parsed.port or self.port
-        except ValueError:
-            port = self.port
-        netloc = parsed.netloc if port is None else '%s:%s' % (parsed.hostname, port)
-
-        url_components = list(parsed)
-        url_components[1] = netloc
-        ret = urlparse.urlunsplit(url_components)
-        return ret.lstrip('/')
-
-
-class BaseHTTPSHandler(BaseHandler):
-    '''
-    Originally derived from python-logstash library
-    Non-blocking request accomplished by FuturesSession, similar
-    to the loggly-python-handler library
-    '''
-    def _add_auth_information(self):
-        if self.message_type == 'logstash':
-            if not self.username:
-                # Logstash authentication not enabled
-                return
-            logstash_auth = requests.auth.HTTPBasicAuth(self.username, self.password)
-            self.session.auth = logstash_auth
-        elif self.message_type == 'splunk':
-            auth_header = "Splunk %s" % self.password
-            headers = {
-                "Authorization": auth_header,
-                "Content-Type": "application/json"
-            }
-            self.session.headers.update(headers)
-
-    def __init__(self, fqdn=False, message_type=None, username=None, password=None,
-                 tcp_timeout=5, verify_cert=True, **kwargs):
-        self.fqdn = fqdn
-        self.message_type = message_type
-        self.username = username
-        self.password = password
-        self.tcp_timeout = tcp_timeout
-        self.verify_cert = verify_cert
-        super(BaseHTTPSHandler, self).__init__(**kwargs)
-        self.session = FuturesSession(executor=VerboseThreadPoolExecutor(
-            max_workers=2  # this is the default used by requests_futures
-        ))
-        self._add_auth_information()
-
-    def _get_post_kwargs(self, payload_input):
-        if self.message_type == 'splunk':
-            # Splunk needs data nested under key "event"
-            if not isinstance(payload_input, dict):
-                payload_input = json.loads(payload_input)
-            payload_input = {'event': payload_input}
-        if isinstance(payload_input, dict):
-            payload_str = json.dumps(payload_input)
-        else:
-            payload_str = payload_input
-        kwargs = dict(data=payload_str, background_callback=unused_callback,
-                      timeout=self.tcp_timeout)
-        if self.verify_cert is False:
-            kwargs['verify'] = False
-        return kwargs
-
-
-    def _send(self, payload):
-        """See:
-            https://docs.python.org/3/library/concurrent.futures.html#future-objects
-            http://pythonhosted.org/futures/
-        """
-        return self.session.post(self._get_host(scheme='https'),
-                                 **self._get_post_kwargs(payload))
-
-
-def _encode_payload_for_socket(payload):
-    encoded_payload = payload
-    if isinstance(encoded_payload, dict):
-        encoded_payload = json.dumps(encoded_payload, ensure_ascii=False)
-    if isinstance(encoded_payload, str):
-        encoded_payload = encoded_payload.encode('utf-8')
-    return encoded_payload
-
-
-class TCPHandler(BaseHandler):
-    def __init__(self, tcp_timeout=5, **kwargs):
-        self.tcp_timeout = tcp_timeout
-        super(TCPHandler, self).__init__(**kwargs)
-
-    def _send(self, payload):
-        payload = _encode_payload_for_socket(payload)
-        sok = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        try:
-            sok.connect((self._get_host(hostname_only=True), self.port or 0))
-            sok.setblocking(0)
-            _, ready_to_send, _ = select.select([], [sok], [], float(self.tcp_timeout))
-            if len(ready_to_send) == 0:
-                ret = SocketResult(False, "Socket currently busy, failed to send message")
-                logger.warning(ret.reason)
-            else:
-                sok.send(payload)
-                ret = SocketResult(True)  # success!
-        except Exception as e:
-            ret = SocketResult(False, "Error sending message from %s: %s" %
-                               (TCPHandler.__name__,
-                                ' '.join(str(arg) for arg in e.args)))
-            logger.exception(ret.reason)
-        finally:
-            sok.close()
-        return ret
-
-
-class UDPHandler(BaseHandler):
-    message = "Cannot determine if UDP messages are received."
-
-    def __init__(self, **kwargs):
-        super(UDPHandler, self).__init__(**kwargs)
-        self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-
-    def _send(self, payload):
-        payload = _encode_payload_for_socket(payload)
-        self.socket.sendto(payload, (self._get_host(hostname_only=True), self.port or 0))
-        return SocketResult(True, reason=self.message)
-
-
-class AWXNullHandler(logging.NullHandler):
-    '''
-    Only additional this does is accept arbitrary __init__ params because
-    the proxy handler does not (yet) work with arbitrary handler classes
-    '''
-    def __init__(self, *args, **kwargs):
-        super(AWXNullHandler, self).__init__()
-
-
-HANDLER_MAPPING = {
-    'https': BaseHTTPSHandler,
-    'tcp': TCPHandler,
-    'udp': UDPHandler,
-}
-
-
-TTLCache = cachetools.TTLCache
-
-if 'py.test' in os.environ.get('_', ''):
-    # don't cache settings in unit tests
-    class TTLCache(TTLCache):
-
-        def __getitem__(self, item):
-            raise KeyError()
-
-
-class AWXProxyHandler(logging.Handler):
-    '''
-    Handler specific to the AWX external logging feature
-
-    Will dynamically create a handler specific to the configured
-    protocol, and will create a new one automatically on setting change
-
-    Managing parameters:
-    All parameters will get their value from settings as a default
-    if the parameter was either provided on init, or set manually,
-    this value will take precedence.
-    Parameters match same parameters in the actualized handler classes.
-    '''
-
-    thread_local = threading.local()
-    _auditor = None
-
-    def __init__(self, **kwargs):
-        # TODO: process 'level' kwarg
-        super(AWXProxyHandler, self).__init__(**kwargs)
-        self._handler = None
-        self._old_kwargs = {}
-
-    @property
-    def auditor(self):
-        if not self._auditor:
-            self._auditor = logging.handlers.RotatingFileHandler(
-                filename='/var/log/tower/external.log',
-                maxBytes=1024 * 1024 * 50, # 50 MB
-                backupCount=5,
-            )
-
-            class WritableLogstashFormatter(LogstashFormatter):
-                @classmethod
-                def serialize(cls, message):
-                    return json.dumps(message)
-
-            self._auditor.setFormatter(WritableLogstashFormatter())
-        return self._auditor
-
-    def get_handler_class(self, protocol):
-        return HANDLER_MAPPING.get(protocol, AWXNullHandler)
-
-    @cachetools.cached(cache=TTLCache(maxsize=1, ttl=3), key=lambda *args, **kw: 'get_handler')
-    def get_handler(self, custom_settings=None, force_create=False):
-        new_kwargs = {}
-        use_settings = custom_settings or settings
-        for field_name, setting_name in PARAM_NAMES.items():
-            val = getattr(use_settings, setting_name, None)
-            if val is None:
-                continue
-            new_kwargs[field_name] = val
-        if new_kwargs == self._old_kwargs and self._handler and (not force_create):
-            # avoids re-creating session objects, and other such things
-            return self._handler
-        self._old_kwargs = new_kwargs.copy()
-        # TODO: remove any kwargs no applicable to that particular handler
-        protocol = new_kwargs.pop('protocol', None)
-        HandlerClass = self.get_handler_class(protocol)
-        # cleanup old handler and make new one
-        if self._handler:
-            self._handler.close()
-        logger.debug('Creating external log handler due to startup or settings change.')
-        self._handler = HandlerClass(**new_kwargs)
-        if self.formatter:
-            # self.format(record) is called inside of emit method
-            # so not safe to assume this can be handled within self
-            self._handler.setFormatter(self.formatter)
-        return self._handler
-
-    @cachetools.cached(cache=TTLCache(maxsize=1, ttl=3), key=lambda *args, **kw: 'should_audit')
-    def should_audit(self):
-        return settings.LOG_AGGREGATOR_AUDIT
-
-    def emit(self, record):
-        if AWXProxyHandler.thread_local.enabled:
-            actual_handler = self.get_handler()
-            if self.should_audit():
-                self.auditor.setLevel(settings.LOG_AGGREGATOR_LEVEL)
-                self.auditor.emit(record)
-            return actual_handler.emit(record)
-
-    def perform_test(self, custom_settings):
-        """
-        Tests logging connectivity for given settings module.
-        @raises LoggingConnectivityException
-        """
-        handler = self.get_handler(custom_settings=custom_settings, force_create=True)
-        handler.setFormatter(LogstashFormatter())
-        logger = logging.getLogger(__file__)
-        fn, lno, func, _ = logger.findCaller()
-        record = logger.makeRecord('awx', 10, fn, lno,
-                                   'AWX Connection Test', tuple(),
-                                   None, func)
-        futures = handler.emit(record)
-        for future in futures:
-            try:
-                resp = future.result()
-                if not resp.ok:
-                    if isinstance(resp, SocketResult):
-                        raise LoggingConnectivityException(
-                            'Socket error: {}'.format(resp.reason or '')
-                        )
-                    else:
-                        raise LoggingConnectivityException(
-                            ': '.join([str(resp.status_code), resp.reason or ''])
-                        )
-            except RequestException as e:
-                raise LoggingConnectivityException(str(e))
-
-    @classmethod
-    def disable(cls):
-        cls.thread_local.enabled = False
-
-
-AWXProxyHandler.thread_local.enabled = True
-
-
 ColorHandler = logging.StreamHandler
 
 if settings.COLOR_LOGS is True:
diff --git a/awx/main/utils/reload.py b/awx/main/utils/reload.py
index bdfcc0dcc9..7c8ff4c999 100644
--- a/awx/main/utils/reload.py
+++ b/awx/main/utils/reload.py
@@ -11,18 +11,13 @@ from django.conf import settings
 logger = logging.getLogger('awx.main.utils.reload')
 
 
-def _supervisor_service_command(command, communicate=True):
+def supervisor_service_command(command, service='*', communicate=True):
     '''
     example use pattern of supervisorctl:
     # supervisorctl restart tower-processes:receiver tower-processes:factcacher
     '''
-    group_name = 'tower-processes'
-    if settings.DEBUG:
-        group_name = 'awx-processes'
     args = ['supervisorctl']
-    if settings.DEBUG:
-        args.extend(['-c', '/supervisor.conf'])
-    args.extend([command, '{}:*'.format(group_name)])
+    args.extend([command, ':'.join(['tower-processes', service])])
     logger.debug('Issuing command to {} services, args={}'.format(command, args))
     supervisor_process = subprocess.Popen(args, stdin=subprocess.PIPE,
                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -41,4 +36,4 @@ def _supervisor_service_command(command, communicate=True):
 
 def stop_local_services(communicate=True):
     logger.warn('Stopping services on this node in response to user action')
-    _supervisor_service_command(command='stop', communicate=communicate)
+    supervisor_service_command(command='stop', communicate=communicate)
diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
index 53a72f369f..a53e65bfad 100644
--- a/awx/settings/defaults.py
+++ b/awx/settings/defaults.py
@@ -1011,8 +1011,9 @@ LOGGING = {
             'formatter': 'simple',
         },
         'external_logger': {
-            'class': 'awx.main.utils.handlers.AWXProxyHandler',
+            'class': 'logging.handlers.SysLogHandler',
             'formatter': 'json',
+            'address': ('localhost', 51414),
             'filters': ['external_log_enabled', 'dynamic_level_filter'],
         },
         'tower_warnings': {
diff --git a/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-logging.form.js b/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-logging.form.js
index 388cfb00f8..dc57e3ece5 100644
--- a/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-logging.form.js
+++ b/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-logging.form.js
@@ -78,7 +78,7 @@
                  ngClick: 'vm.testLogging()',
                  label: i18n._('Test'),
                  class: 'btn-primary',
-                 ngDisabled: 'configuration_logging_template_form.$invalid'
+                 ngDisabled: 'configuration_logging_template_form.$pending'
              },
              cancel: {
                  ngClick: 'vm.formCancel()',
diff --git a/docs/licenses/requests-futures.txt b/docs/licenses/requests-futures.txt
deleted file mode 100644
index 2a2a0c6288..0000000000
--- a/docs/licenses/requests-futures.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-Copyright 2013 Ross McFarland
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/requirements/requirements.in b/requirements/requirements.in
index a1869c8978..c847496844 100644
--- a/requirements/requirements.in
+++ b/requirements/requirements.in
@@ -42,7 +42,6 @@ social-auth-core==3.3.1  # see UPGRADE BLOCKERs
 social-auth-app-django==3.1.0  # see UPGRADE BLOCKERs
 redis
 requests
-requests-futures  # see library notes
 slackclient==1.1.2  # see UPGRADE BLOCKERs
 tacacs_plus==1.0  # UPGRADE BLOCKER: auth does not work with later versions
 twilio
diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile
index 0ec22e499d..f59f466a6f 100644
--- a/tools/docker-compose/Dockerfile
+++ b/tools/docker-compose/Dockerfile
@@ -102,6 +102,10 @@ RUN cd /usr/local/bin && \
     curl -L https://github.com/openshift/origin/releases/download/v3.9.0/openshift-origin-client-tools-v3.9.0-191fece-linux-64bit.tar.gz | \
     tar -xz --strip-components=1 --wildcards --no-anchored 'oc'
 
+ADD tools/docker-compose/rsyslog.repo /etc/yum.repos.d/
+RUN yum install -y rsyslog-omhttp
+RUN echo '$IncludeConfig /var/lib/awx/rsyslog.conf' >> /etc/rsyslog.conf
+
 RUN dnf -y clean all && rm -rf /root/.cache
 
 # https://github.com/ansible/awx/issues/5224
diff --git a/tools/docker-compose/rsyslog.repo b/tools/docker-compose/rsyslog.repo
new file mode 100644
index 0000000000..4cc2a35a42
--- /dev/null
+++ b/tools/docker-compose/rsyslog.repo
@@ -0,0 +1,7 @@
+[rsyslog_v8]
+name=Adiscon CentOS-$releasever - local packages for $basearch
+baseurl=http://rpms.adiscon.com/v8-stable/epel-$releasever/$basearch
+enabled=1
+gpgcheck=0
+gpgkey=http://rpms.adiscon.com/RPM-GPG-KEY-Adiscon
+protect=1
diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf
index 6831d80203..2a39e0fba6 100644
--- a/tools/docker-compose/supervisor.conf
+++ b/tools/docker-compose/supervisor.conf
@@ -71,8 +71,20 @@ redirect_stderr=true
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
+[program:awx-rsyslogd]
+command = rsyslogd -nd -i /awx_devel/rsyslog.pid
+autostart = true
+autorestart = true
+stopwaitsecs = 1
+stopsignal=KILL
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/tmp/ryan
+stdout_logfile_maxbytes=0
+
 [group:tower-processes]
-programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast
+programs=awx-dispatcher,awx-receiver,awx-runworker,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast,awx-rsyslogd
 priority=5
 
 [unix_http_server]

From 955d57bce626a394217e66210d866c70d352f4c5 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Tue, 14 Jan 2020 10:15:50 -0500
Subject: [PATCH 02/37] Upstream rsyslog packaging changes  - add rsyslog repo
 to Dockerfile for AWX installation  - Update Library Notes for
 requests-futures removal

---
 installer/roles/image_build/files/rsyslog.repo      | 7 +++++++
 installer/roles/image_build/tasks/main.yml          | 7 +++++++
 installer/roles/image_build/templates/Dockerfile.j2 | 4 ++++
 requirements/README.md                              | 4 ----
 4 files changed, 18 insertions(+), 4 deletions(-)
 create mode 100644 installer/roles/image_build/files/rsyslog.repo

diff --git a/installer/roles/image_build/files/rsyslog.repo b/installer/roles/image_build/files/rsyslog.repo
new file mode 100644
index 0000000000..4cc2a35a42
--- /dev/null
+++ b/installer/roles/image_build/files/rsyslog.repo
@@ -0,0 +1,7 @@
+[rsyslog_v8]
+name=Adiscon CentOS-$releasever - local packages for $basearch
+baseurl=http://rpms.adiscon.com/v8-stable/epel-$releasever/$basearch
+enabled=1
+gpgcheck=0
+gpgkey=http://rpms.adiscon.com/RPM-GPG-KEY-Adiscon
+protect=1
diff --git a/installer/roles/image_build/tasks/main.yml b/installer/roles/image_build/tasks/main.yml
index 4694dcd15f..51e9b239ab 100644
--- a/installer/roles/image_build/tasks/main.yml
+++ b/installer/roles/image_build/tasks/main.yml
@@ -137,6 +137,13 @@
     mode: '0700'
   delegate_to: localhost
 
+- name: Stage rsyslog.repo
+  copy:
+    src: rsyslog.repo
+    dest: "{{ docker_base_path }}/rsyslog.repo"
+    mode: '0700'
+  delegate_to: localhost
+
 - name: Stage supervisor.conf
   copy:
     src: supervisor.conf
diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2
index fa1e3cc1d7..44668ec6b8 100644
--- a/installer/roles/image_build/templates/Dockerfile.j2
+++ b/installer/roles/image_build/templates/Dockerfile.j2
@@ -97,6 +97,10 @@ RUN cd /usr/local/bin && \
     curl -L https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz | \
     tar -xz --strip-components=1 --wildcards --no-anchored 'oc'
 
+ADD rsyslog.repo /etc/yum.repos.d/
+RUN yum install -y rsyslog-omhttp
+RUN echo '$IncludeConfig /var/lib/awx/rsyslog.conf' >> /etc/rsyslog.conf
+
 # Pre-create things that we need to write to
 RUN for dir in /home/awx /var/log/tower /var/log/nginx /var/lib/nginx; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
diff --git a/requirements/README.md b/requirements/README.md
index 3afa92434e..4ce399a7e9 100644
--- a/requirements/README.md
+++ b/requirements/README.md
@@ -145,7 +145,3 @@ in the top-level Makefile.
 Version 4.8 makes us a little bit nervous with changes to `searchwindowsize` https://github.com/pexpect/pexpect/pull/579/files
 Pin to `pexpect==4.7.x` until we have more time to move to `4.8` and test.
 
-### requests-futures
-
-This can be removed when a solution for the external log queuing is ready.
-https://github.com/ansible/awx/pull/5092

From 4cd0d60711c90b1b5b0e3cff7aeffc9fec1301c7 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Fri, 24 Jan 2020 13:56:01 -0500
Subject: [PATCH 03/37] Properly handle logger paths and https/http
 configuration  - log aggregator url paths were not being passed to rsyslog  -
 http log services like loggly will now truly use http and port 80  - add
 rsyslog.pid to .gitignore

---
 .gitignore                         | 1 +
 awx/main/utils/external_logging.py | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index ac443e2acd..f772abd40c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,6 +31,7 @@ awx/ui/templates/ui/installing.html
 awx/ui_next/node_modules/
 awx/ui_next/coverage/
 awx/ui_next/build/locales/_build
+rsyslog.pid
 /tower-license
 /tower-license/**
 tools/prometheus/data
diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 26c13b195b..a30003ef5d 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -22,7 +22,7 @@ def reconfigure_rsyslog():
 
             host = parsed.hostname
             try:
-                port = parsed.port or settings.LOG_AGGREGATOR_PORT
+                port = parsed.port
             except ValueError:
                 port = settings.LOG_AGGREGATOR_PORT
 
@@ -36,6 +36,9 @@ def reconfigure_rsyslog():
             # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst
             ssl = "on" if parsed.scheme == 'https' else "off"
             skip_verify = "off" if settings.LOG_AGGREGATOR_VERIFY_CERT else "on"
+            if not port:
+                port = 443 if parsed.scheme == 'https' else 80
+
             params = [
                 'type="omhttp"',
                 f'server="{host}"',
@@ -45,7 +48,10 @@ def reconfigure_rsyslog():
                 'action.resumeRetryCount="-1"',
                 'template="awx"',
                 'errorfile="/var/log/tower/external.err"',
+                'healthchecktimeout="20000"',
             ]
+            if parsed.path:
+                params.append(f'restpath="{parsed.path[1:]}"')
             username = getattr(settings, 'LOG_AGGREGATOR_USERNAME', '')
             password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '')
             if username:

From f8afae308a96c0bc38d80c27be8f29ff25fb26d4 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Tue, 28 Jan 2020 10:42:09 -0500
Subject: [PATCH 04/37] Add rsyslog to supervisor for the task container  - Add
 proper paths for rsyslog's supervisor logs  - Do not enable debug mode for
 rsyslogd  - Include system rsyslog.conf, and specify tower logging conf when 
   starting rsyslog.

---
 awx/main/utils/external_logging.py                 |  1 +
 .../roles/image_build/files/supervisor_task.conf   | 14 +++++++++++++-
 tools/docker-compose/supervisor.conf               |  4 ++--
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index a30003ef5d..bef6dac151 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -28,6 +28,7 @@ def reconfigure_rsyslog():
 
         parts = []
         parts.extend([
+            '$IncludeConfig /etc/rsyslog.conf',
             '$ModLoad imudp',
             '$UDPServerRun 51414',
             'template(name="awx" type="string" string="%msg%")',
diff --git a/installer/roles/image_build/files/supervisor_task.conf b/installer/roles/image_build/files/supervisor_task.conf
index a0100980b2..9acf19f7e9 100644
--- a/installer/roles/image_build/files/supervisor_task.conf
+++ b/installer/roles/image_build/files/supervisor_task.conf
@@ -26,8 +26,20 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
+[program:awx-rsyslogd]
+command = rsyslogd -n -i /awx_devel/rsyslog.pid
+autostart = true
+autorestart = true
+stopwaitsecs = 1
+stopsignal=KILL
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/stderr
+stdout_logfile_maxbytes=0
+
 [group:tower-processes]
-programs=dispatcher,callback-receiver
+programs=dispatcher,callback-receiver,awx-rsyslogd
 priority=5
 
 # TODO: Exit Handler
diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf
index 2a39e0fba6..f5416c4b26 100644
--- a/tools/docker-compose/supervisor.conf
+++ b/tools/docker-compose/supervisor.conf
@@ -72,7 +72,7 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
 [program:awx-rsyslogd]
-command = rsyslogd -nd -i /awx_devel/rsyslog.pid
+command = rsyslogd -n -i /awx_devel/rsyslog.pid
 autostart = true
 autorestart = true
 stopwaitsecs = 1
@@ -80,7 +80,7 @@ stopsignal=KILL
 stopasgroup=true
 killasgroup=true
 redirect_stderr=true
-stdout_logfile=/tmp/ryan
+stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
 [group:tower-processes]

From c0af3c537be66c19b87e4617ee170c302c0d7f17 Mon Sep 17 00:00:00 2001
From: Shane McDonald <me@shanemcd.com>
Date: Tue, 25 Feb 2020 19:55:14 -0500
Subject: [PATCH 05/37] Configure rsyslog to listen over a unix domain socket
 instead of a port  - Add a placeholder rsyslog.conf so it doesn't fail on
 start  - Create access restricted directory for unix socket to be created in 
 - Create RSyslogHandler to exit early when logging socket doesn't exist  -
 Write updated logging settings when dispatcher comes up and restart rsyslog
 so they  take effect  - Move rsyslogd to the web container and create rpc
 supervisor.sock  - Add env var for supervisor.conf path

---
 awx/main/tasks.py                              |  3 +++
 awx/main/utils/external_logging.py             | 10 ++++------
 awx/main/utils/handlers.py                     | 10 ++++++++++
 awx/main/utils/reload.py                       |  6 ++++++
 awx/settings/defaults.py                       |  5 +++--
 .../roles/image_build/files/supervisor.conf    | 18 +++++++++++++++---
 .../roles/image_build/templates/Dockerfile.j2  |  3 +--
 .../kubernetes/templates/deployment.yml.j2     | 14 ++++++++++++++
 .../templates/docker-compose.yml.j2            |  8 ++++++++
 tools/docker-compose/Dockerfile                | 11 ++++++++++-
 tools/docker-compose/supervisor.conf           |  2 +-
 11 files changed, 75 insertions(+), 15 deletions(-)

diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index 3116351644..ebf29bbe97 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -141,6 +141,9 @@ def dispatch_startup():
     # and Tower fall out of use/support, we can probably just _assume_ that
     # everybody has moved to bigint, and remove this code entirely
     enforce_bigint_pk_migration()
+    
+    # Update Tower's rsyslog.conf file based on loggins settings in the db
+    reconfigure_rsyslog()
 
 
 def inform_cluster_of_shutdown():
diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index bef6dac151..1d373293db 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -7,6 +7,7 @@ from awx.main.utils.reload import supervisor_service_command
 
 def reconfigure_rsyslog():
     tmpl = ''
+    parts = ['$IncludeConfig /etc/rsyslog.conf']
     if settings.LOG_AGGREGATOR_ENABLED:
         host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
         port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
@@ -26,11 +27,8 @@ def reconfigure_rsyslog():
             except ValueError:
                 port = settings.LOG_AGGREGATOR_PORT
 
-        parts = []
         parts.extend([
-            '$IncludeConfig /etc/rsyslog.conf',
-            '$ModLoad imudp',
-            '$UDPServerRun 51414',
+            'input(type="imuxsock" Socket="/var/run/tower/sockets/rsyslog.sock" unlink="on")',
             'template(name="awx" type="string" string="%msg%")',
         ])
         if protocol.startswith('http'):
@@ -65,8 +63,8 @@ def reconfigure_rsyslog():
             parts.append(
                 f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
             )
-        tmpl = '\n'.join(parts)
 
-    with open('/var/lib/awx/rsyslog.conf', 'w') as f:
+    tmpl = '\n'.join(parts)
+    with open('/var/lib/awx/rsyslog/rsyslog.conf', 'w') as f:
         f.write(tmpl + '\n')
     supervisor_service_command(command='restart', service='awx-rsyslogd')
diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py
index 0640c14921..26b4282fd1 100644
--- a/awx/main/utils/handlers.py
+++ b/awx/main/utils/handlers.py
@@ -3,10 +3,20 @@
 
 # Python
 import logging
+import os.path
 
 # Django
 from django.conf import settings
 
+
+class RSysLogHandler(logging.handlers.SysLogHandler):
+
+    def emit(self, msg):
+        if not os.path.exists(settings.LOGGING_SOCK):
+            return
+        return super(RSysLogHandler, self).emit(msg)
+
+
 ColorHandler = logging.StreamHandler
 
 if settings.COLOR_LOGS is True:
diff --git a/awx/main/utils/reload.py b/awx/main/utils/reload.py
index 7c8ff4c999..04868f06d0 100644
--- a/awx/main/utils/reload.py
+++ b/awx/main/utils/reload.py
@@ -4,6 +4,7 @@
 # Python
 import subprocess
 import logging
+import os
 
 # Django
 from django.conf import settings
@@ -17,6 +18,11 @@ def supervisor_service_command(command, service='*', communicate=True):
     # supervisorctl restart tower-processes:receiver tower-processes:factcacher
     '''
     args = ['supervisorctl']
+
+    supervisor_config_path = os.getenv('SUPERVISOR_WEB_CONFIG_PATH', None)
+    if supervisor_config_path:
+        args.extend(['-c', supervisor_config_path])
+
     args.extend([command, ':'.join(['tower-processes', service])])
     logger.debug('Issuing command to {} services, args={}'.format(command, args))
     supervisor_process = subprocess.Popen(args, stdin=subprocess.PIPE,
diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
index a53e65bfad..0207f92b55 100644
--- a/awx/settings/defaults.py
+++ b/awx/settings/defaults.py
@@ -955,6 +955,7 @@ CHANNEL_LAYERS = {
 }
 
 # Logging configuration.
+LOGGING_SOCK = '/var/run/tower/sockets/rsyslog.sock'
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -1011,9 +1012,9 @@ LOGGING = {
             'formatter': 'simple',
         },
         'external_logger': {
-            'class': 'logging.handlers.SysLogHandler',
+            'class': 'awx.main.utils.handlers.RSysLogHandler',
             'formatter': 'json',
-            'address': ('localhost', 51414),
+            'address': LOGGING_SOCK,
             'filters': ['external_log_enabled', 'dynamic_level_filter'],
         },
         'tower_warnings': {
diff --git a/installer/roles/image_build/files/supervisor.conf b/installer/roles/image_build/files/supervisor.conf
index acc1af1d6b..bd49186170 100644
--- a/installer/roles/image_build/files/supervisor.conf
+++ b/installer/roles/image_build/files/supervisor.conf
@@ -46,8 +46,20 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
+[program:awx-rsyslogd]
+command = rsyslogd -n -i /var/run/tower/sockets/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
+autostart = true
+autorestart = true
+stopwaitsecs = 1
+stopsignal=KILL
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/stderr
+stdout_logfile_maxbytes=0
+
 [group:tower-processes]
-programs=nginx,uwsgi,daphne,wsbroadcast
+programs=nginx,uwsgi,daphne,wsbroadcast,awx-rsyslogd
 priority=5
 
 # TODO: Exit Handler
@@ -62,10 +74,10 @@ events=TICK_60
 priority=0
 
 [unix_http_server]
-file=/tmp/supervisor.sock
+file=/var/run/tower/sockets/supervisor.web.sock
 
 [supervisorctl]
-serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
+serverurl=unix:///var/run/tower/sockets/supervisor.web.sock ; use a unix:// URL  for a unix socket
 
 [rpcinterface:supervisor]
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2
index 44668ec6b8..8b829e7976 100644
--- a/installer/roles/image_build/templates/Dockerfile.j2
+++ b/installer/roles/image_build/templates/Dockerfile.j2
@@ -99,10 +99,9 @@ RUN cd /usr/local/bin && \
 
 ADD rsyslog.repo /etc/yum.repos.d/
 RUN yum install -y rsyslog-omhttp
-RUN echo '$IncludeConfig /var/lib/awx/rsyslog.conf' >> /etc/rsyslog.conf
 
 # Pre-create things that we need to write to
-RUN for dir in /home/awx /var/log/tower /var/log/nginx /var/lib/nginx; \
+RUN for dir in /home/awx /var/run/supervisor /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /var/run/nginx.pid; \
diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2
index 869fbb0ffb..cdf414f28b 100644
--- a/installer/roles/kubernetes/templates/deployment.yml.j2
+++ b/installer/roles/kubernetes/templates/deployment.yml.j2
@@ -94,6 +94,10 @@ spec:
           ports:
             - containerPort: 8052
           volumeMounts:
+            - name: sockets
+              mountPath: "/var/run/tower/sockets/"
+            - name: rsyslog
+              mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
               mountPath: "/etc/pki/ca-trust/source/anchors/"
@@ -174,6 +178,10 @@ spec:
             - /usr/bin/launch_awx_task.sh
           imagePullPolicy: Always
           volumeMounts:
+            - name: sockets
+              mountPath: "/var/run/tower/sockets/"
+            - name: rsyslog
+              mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
               mountPath: "/etc/pki/ca-trust/source/anchors/"
@@ -223,6 +231,8 @@ spec:
             - name: {{ kubernetes_deployment_name }}-memcached-socket
               mountPath: "/var/run/memcached"
           env:
+            - name: SUPERVISOR_WEB_CONFIG_PATH
+              value: "/supervisor.conf"
             - name: AWX_SKIP_MIGRATIONS
               value: "1"
             - name: MY_POD_UID
@@ -313,6 +323,10 @@ spec:
 {{ affinity | to_nice_yaml(indent=2) | indent(width=8, indentfirst=True) }}
 {% endif %}
       volumes:
+        - name: sockets
+          emptyDir: {}
+        - name: rsyslog
+          emptyDir: {}
 {% if ca_trust_dir is defined %}
         - name: {{ kubernetes_deployment_name }}-ca-trust-dir
           hostPath:
diff --git a/installer/roles/local_docker/templates/docker-compose.yml.j2 b/installer/roles/local_docker/templates/docker-compose.yml.j2
index e9a26f4416..3dfb449a73 100644
--- a/installer/roles/local_docker/templates/docker-compose.yml.j2
+++ b/installer/roles/local_docker/templates/docker-compose.yml.j2
@@ -20,6 +20,8 @@ services:
     user: root
     restart: unless-stopped
     volumes:
+      - sockets:/var/run/tower/sockets/
+      - rsyslog:/var/lib/awx/rsyslog/
       - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY"
       - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh"
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
@@ -75,6 +77,8 @@ services:
     user: root
     restart: unless-stopped
     volumes:
+      - sockets:/var/run/tower/sockets/
+      - rsyslog:/var/lib/awx/rsyslog/
       - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY"
       - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh"
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
@@ -114,6 +118,7 @@ services:
       http_proxy: {{ http_proxy | default('') }}
       https_proxy: {{ https_proxy | default('') }}
       no_proxy: {{ no_proxy | default('') }}
+      SUPERVISOR_WEB_CONFIG_PATH: '/supervisor.conf'
 
   redis:
     image: {{ redis_image }}
@@ -157,3 +162,6 @@ services:
       https_proxy: {{ https_proxy | default('') }}
       no_proxy: {{ no_proxy | default('') }}
   {% endif %}
+volumes:
+  sockets:
+  rsyslog:
diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile
index f59f466a6f..7fd9b8e74b 100644
--- a/tools/docker-compose/Dockerfile
+++ b/tools/docker-compose/Dockerfile
@@ -104,7 +104,7 @@ RUN cd /usr/local/bin && \
 
 ADD tools/docker-compose/rsyslog.repo /etc/yum.repos.d/
 RUN yum install -y rsyslog-omhttp
-RUN echo '$IncludeConfig /var/lib/awx/rsyslog.conf' >> /etc/rsyslog.conf
+RUN mkdir -p /var/lib/awx/rsyslog/ && echo '$IncludeConfig /etc/rsyslog.conf' >> /var/lib/awx/rsyslog/rsyslog.conf
 
 RUN dnf -y clean all && rm -rf /root/.cache
 
@@ -123,11 +123,20 @@ ADD tools/docker-compose/entrypoint.sh /
 ADD tools/scripts/awx-python /usr/bin/awx-python
 
 # Pre-create things that we need to write to
+<<<<<<< HEAD
 RUN for dir in /var/lib/awx/ /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \
     do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done
+=======
+RUN for dir in /var/lib/awx/rsyslog /var/run/tower/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
+  do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done
+
+RUN for file in /etc/passwd /etc/supervisord.conf \
+  /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \
+  do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done
+>>>>>>> Configure rsyslog to listen over a unix domain socket instead of a port
 
 ENV HOME /var/lib/awx
 ENV PATH="/usr/local/n/versions/node/10.15.0/bin:${PATH}"
diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf
index f5416c4b26..826088c6ad 100644
--- a/tools/docker-compose/supervisor.conf
+++ b/tools/docker-compose/supervisor.conf
@@ -72,7 +72,7 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
 [program:awx-rsyslogd]
-command = rsyslogd -n -i /awx_devel/rsyslog.pid
+command = rsyslogd -n -i /var/run/tower/rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
 autostart = true
 autorestart = true
 stopwaitsecs = 1

From 88ca4b63e657b563d937add14dbe25fb71b07ed9 Mon Sep 17 00:00:00 2001
From: John Mitchell <jmitchel@Johns-MacBook-Pro-46.local>
Date: Mon, 9 Mar 2020 13:39:01 -0400
Subject: [PATCH 06/37] update configure tower in tower test ui for log
 aggregator form

---
 .../forms/settings-form.controller.js         |   6 +-
 .../configuration-system.controller.js        | 122 ++++++++++--------
 .../sub-forms/system-logging.form.js          |   7 +-
 awx/ui/client/src/shared/form-generator.js    |   3 +
 4 files changed, 80 insertions(+), 58 deletions(-)

diff --git a/awx/ui/client/src/configuration/forms/settings-form.controller.js b/awx/ui/client/src/configuration/forms/settings-form.controller.js
index 907f978a93..e78d08baaa 100644
--- a/awx/ui/client/src/configuration/forms/settings-form.controller.js
+++ b/awx/ui/client/src/configuration/forms/settings-form.controller.js
@@ -92,6 +92,7 @@ export default [
         var populateFromApi = function() {
             SettingsService.getCurrentValues()
                 .then(function(data) {
+                    $scope.logAggregatorEnabled = data.LOG_AGGREGATOR_ENABLED;
                     // these two values need to be unnested from the
                     // OAUTH2_PROVIDER key
                     data.ACCESS_TOKEN_EXPIRE_SECONDS = data
@@ -538,8 +539,11 @@ export default [
             var payload = {};
             payload[key] = $scope[key];
             SettingsService.patchConfiguration(payload)
-                .then(function() {
+                .then(function(data) {
                     //TODO consider updating form values with returned data here
+                    if (key === 'LOG_AGGREGATOR_ENABLED') {
+                        $scope.logAggregatorEnabled = data.LOG_AGGREGATOR_ENABLED;
+                    }
                 })
                 .catch(function(data) {
                     //Change back on unsuccessful update
diff --git a/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js b/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
index 1b78d85685..4178b2f929 100644
--- a/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
+++ b/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
@@ -17,7 +17,7 @@ export default [
     'ProcessErrors',
     'ngToast',
     '$filter',
-    function(
+    function (
         $rootScope, $scope, $stateParams,
         systemActivityStreamForm,
         systemLoggingForm,
@@ -41,8 +41,8 @@ export default [
             formTracker.setCurrentSystem(activeSystemForm);
         }
 
-        var activeForm = function(tab) {
-            if(!_.get($scope.$parent, [formTracker.currentFormName(), '$dirty'])) {
+        var activeForm = function (tab) {
+            if (!_.get($scope.$parent, [formTracker.currentFormName(), '$dirty'])) {
                 systemVm.activeSystemForm = tab;
                 formTracker.setCurrentSystem(systemVm.activeSystemForm);
             } else {
@@ -52,7 +52,7 @@ export default [
                     label: i18n._('Discard changes'),
                     "class": "btn Form-cancelButton",
                     "id": "formmodal-cancel-button",
-                    onClick: function() {
+                    onClick: function () {
                         $scope.$parent.vm.populateFromApi();
                         $scope.$parent[formTracker.currentFormName()].$setPristine();
                         systemVm.activeSystemForm = tab;
@@ -61,15 +61,15 @@ export default [
                     }
                 }, {
                     label: i18n._('Save changes'),
-                    onClick: function() {
+                    onClick: function () {
                         $scope.$parent.vm.formSave()
-                        .then(function() {
-                            $scope.$parent[formTracker.currentFormName()].$setPristine();
-                            $scope.$parent.vm.populateFromApi();
-                            systemVm.activeSystemForm = tab;
-                            formTracker.setCurrentSystem(systemVm.activeSystemForm);
-                            $('#FormModal-dialog').dialog('close');
-                        });
+                            .then(function () {
+                                $scope.$parent[formTracker.currentFormName()].$setPristine();
+                                $scope.$parent.vm.populateFromApi();
+                                systemVm.activeSystemForm = tab;
+                                formTracker.setCurrentSystem(systemVm.activeSystemForm);
+                                $('#FormModal-dialog').dialog('close');
+                            });
                     },
                     "class": "btn btn-primary",
                     "id": "formmodal-save-button"
@@ -80,9 +80,9 @@ export default [
         };
 
         var dropdownOptions = [
-            {label: i18n._('Misc. System'), value: 'misc'},
-            {label: i18n._('Activity Stream'), value: 'activity_stream'},
-            {label: i18n._('Logging'), value: 'logging'},
+            { label: i18n._('Misc. System'), value: 'misc' },
+            { label: i18n._('Activity Stream'), value: 'activity_stream' },
+            { label: i18n._('Logging'), value: 'logging' },
         ];
 
         var systemForms = [{
@@ -97,14 +97,14 @@ export default [
         }];
 
         var forms = _.map(systemForms, 'formDef');
-        _.each(forms, function(form) {
+        _.each(forms, function (form) {
             var keys = _.keys(form.fields);
-            _.each(keys, function(key) {
-                if($scope.configDataResolve[key].type === 'choice') {
+            _.each(keys, function (key) {
+                if ($scope.configDataResolve[key].type === 'choice') {
                     // Create options for dropdowns
                     var optionsGroup = key + '_options';
                     $scope.$parent.$parent[optionsGroup] = [];
-                    _.each($scope.configDataResolve[key].choices, function(choice){
+                    _.each($scope.configDataResolve[key].choices, function (choice) {
                         $scope.$parent.$parent[optionsGroup].push({
                             name: choice[0],
                             label: choice[1],
@@ -121,7 +121,7 @@ export default [
         function addFieldInfo(form, key) {
             _.extend(form.fields[key], {
                 awPopOver: ($scope.configDataResolve[key].defined_in_file) ?
-                    null: $scope.configDataResolve[key].help_text,
+                    null : $scope.configDataResolve[key].help_text,
                 label: $scope.configDataResolve[key].label,
                 name: key,
                 toggleSource: key,
@@ -138,7 +138,7 @@ export default [
 
         $scope.$parent.$parent.parseType = 'json';
 
-        _.each(systemForms, function(form) {
+        _.each(systemForms, function (form) {
             generator.inject(form.formDef, {
                 id: form.id,
                 mode: 'edit',
@@ -150,37 +150,37 @@ export default [
 
         var dropdownRendered = false;
 
-        $scope.$on('populated', function() {
+        $scope.$on('populated', function () {
             populateLogAggregator(false);
         });
 
-        $scope.$on('LOG_AGGREGATOR_TYPE_populated', function(e, data, flag) {
+        $scope.$on('LOG_AGGREGATOR_TYPE_populated', function (e, data, flag) {
             populateLogAggregator(flag);
         });
 
-        $scope.$on('LOG_AGGREGATOR_PROTOCOL_populated', function(e, data, flag) {
+        $scope.$on('LOG_AGGREGATOR_PROTOCOL_populated', function (e, data, flag) {
             populateLogAggregator(flag);
         });
 
-        function populateLogAggregator(flag){
+        function populateLogAggregator(flag) {
 
-            if($scope.$parent.$parent.LOG_AGGREGATOR_TYPE !== null) {
+            if ($scope.$parent.$parent.LOG_AGGREGATOR_TYPE !== null) {
                 $scope.$parent.$parent.LOG_AGGREGATOR_TYPE = _.find($scope.$parent.$parent.LOG_AGGREGATOR_TYPE_options, { value: $scope.$parent.$parent.LOG_AGGREGATOR_TYPE });
             }
 
-            if($scope.$parent.$parent.LOG_AGGREGATOR_PROTOCOL !== null) {
+            if ($scope.$parent.$parent.LOG_AGGREGATOR_PROTOCOL !== null) {
                 $scope.$parent.$parent.LOG_AGGREGATOR_PROTOCOL = _.find($scope.$parent.$parent.LOG_AGGREGATOR_PROTOCOL_options, { value: $scope.$parent.$parent.LOG_AGGREGATOR_PROTOCOL });
             }
 
-            if($scope.$parent.$parent.LOG_AGGREGATOR_LEVEL !== null) {
+            if ($scope.$parent.$parent.LOG_AGGREGATOR_LEVEL !== null) {
                 $scope.$parent.$parent.LOG_AGGREGATOR_LEVEL = _.find($scope.$parent.$parent.LOG_AGGREGATOR_LEVEL_options, { value: $scope.$parent.$parent.LOG_AGGREGATOR_LEVEL });
             }
 
-            if(flag !== undefined){
+            if (flag !== undefined) {
                 dropdownRendered = flag;
             }
 
-            if(!dropdownRendered) {
+            if (!dropdownRendered) {
                 dropdownRendered = true;
                 CreateSelect2({
                     element: '#configuration_logging_template_LOG_AGGREGATOR_TYPE',
@@ -193,33 +193,45 @@ export default [
             }
         }
 
-        $scope.$parent.vm.testLogging = function() {
-            Rest.setUrl("/api/v2/settings/logging/test/");
-            Rest.post($scope.$parent.vm.getFormPayload())
-                .then(() => {
-                    ngToast.success({
-                        content: `<i class="fa fa-check-circle
-                            Toast-successIcon"></i>` +
-                                i18n._('Log aggregator test successful.')
-                    });
-                })
-                .catch(({data, status}) => {
-                    if (status === 500) {
-                        ngToast.danger({
-                            content: '<i class="fa fa-exclamation-triangle Toast-successIcon"></i>' +
-                                i18n._('Log aggregator test failed. <br> Detail: ') + $filter('sanitize')(data.error),
-                            additionalClasses: "LogAggregator-failedNotification"
+        $scope.$watchGroup(['configuration_logging_template_form.$pending', 'configuration_logging_template_form.$dirty', '!logAggregatorEnabled'], (vals) => {
+            if (vals.some(val => val === true)) {
+                $scope.$parent.vm.disableTestButton = true;
+                $scope.$parent.vm.testTooltip = i18n._('Save and enable log aggregation before testing the log aggregator.');
+            } else {
+                $scope.$parent.vm.disableTestButton = false;
+                $scope.$parent.vm.testTooltip = i18n._('Send a test response to the configured log aggregator.');
+            }
+        });
+
+        $scope.$parent.vm.testLogging = function () {
+            if (!$scope.$parent.vm.disableTestButton) {
+                Rest.setUrl("/api/v2/settings/logging/test/");
+                Rest.post({})
+                    .then(() => {
+                        ngToast.success({
+                            content: `<i class="fa fa-check-circle
+                                Toast-successIcon"></i>` +
+                                i18n._('Log aggregator test sent successfully.')
                         });
-                    } else {
-                        ProcessErrors($scope, data, status, null,
-                            {
-                                hdr: i18n._('Error!'),
-                                msg: i18n._('There was an error testing the ' +
-                                    'log aggregator. Returned status: ') +
-                                    status
+                    })
+                    .catch(({ data, status }) => {
+                        if (status === 500) {
+                            ngToast.danger({
+                                content: '<i class="fa fa-exclamation-triangle Toast-successIcon"></i>' +
+                                    i18n._('Log aggregator test failed. <br> Detail: ') + $filter('sanitize')(data.error),
+                                additionalClasses: "LogAggregator-failedNotification"
                             });
-                    }
-                });
+                        } else {
+                            ProcessErrors($scope, data, status, null,
+                                {
+                                    hdr: i18n._('Error!'),
+                                    msg: i18n._('There was an error testing the ' +
+                                        'log aggregator. Returned status: ') +
+                                        status
+                                });
+                        }
+                    });
+            }
         };
 
         angular.extend(systemVm, {
diff --git a/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-logging.form.js b/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-logging.form.js
index dc57e3ece5..9febbc4363 100644
--- a/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-logging.form.js
+++ b/awx/ui/client/src/configuration/forms/system-form/sub-forms/system-logging.form.js
@@ -75,10 +75,13 @@
                  class: 'Form-resetAll'
              },
              testLogging: {
+                 ngClass: "{'Form-button--disabled': vm.disableTestButton}",
                  ngClick: 'vm.testLogging()',
                  label: i18n._('Test'),
-                 class: 'btn-primary',
-                 ngDisabled: 'configuration_logging_template_form.$pending'
+                 class: 'Form-primaryButton',
+                 awToolTip: '{{vm.testTooltip}}',
+                 dataTipWatch: 'vm.testTooltip',
+                 dataPlacement: 'top',
              },
              cancel: {
                  ngClick: 'vm.formCancel()',
diff --git a/awx/ui/client/src/shared/form-generator.js b/awx/ui/client/src/shared/form-generator.js
index 1fa3f616a2..b5f515537b 100644
--- a/awx/ui/client/src/shared/form-generator.js
+++ b/awx/ui/client/src/shared/form-generator.js
@@ -1690,6 +1690,9 @@ angular.module('FormGenerator', [GeneratorHelpers.name, 'Utilities', listGenerat
                                     if (button.ngClick) {
                                         html += this.attr(button, 'ngClick');
                                     }
+                                    if (button.ngClass) {
+                                        html += this.attr(button, 'ngClass');
+                                    }
                                     if (button.ngDisabled) {
                                         ngDisabled = (button.ngDisabled===true) ? `${this.form.name}_form.$invalid ||  ${this.form.name}_form.$pending`: button.ngDisabled;
                                         if (btn !== 'reset') {

From 7040fcfd888c7e0f3e5e600e5d762c290a48e94d Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Thu, 12 Mar 2020 13:39:20 -0400
Subject: [PATCH 07/37] Fix container rsyslog dir permissions

---
 tools/docker-compose/Dockerfile | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile
index 7fd9b8e74b..3cf89997b3 100644
--- a/tools/docker-compose/Dockerfile
+++ b/tools/docker-compose/Dockerfile
@@ -123,20 +123,19 @@ ADD tools/docker-compose/entrypoint.sh /
 ADD tools/scripts/awx-python /usr/bin/awx-python
 
 # Pre-create things that we need to write to
-<<<<<<< HEAD
-RUN for dir in /var/lib/awx/ /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
+RUN for dir in /var/lib/awx/rsyslog /var/run/tower/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \
     do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done
-=======
-RUN for dir in /var/lib/awx/rsyslog /var/run/tower/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
-  do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done
 
-RUN for file in /etc/passwd /etc/supervisord.conf \
-  /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \
-  do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done
->>>>>>> Configure rsyslog to listen over a unix domain socket instead of a port
+
+RUN chmod -R 0775 /var/lib/awx /var/lib/awx/rsyslog
+RUN chmod -R 0770 /var/lib/rsyslog                    # needed, or else: rsyslogd: imjournal: open on state file `/var/lib/rsyslog/imjournal.state' failed 
+ADD tools/docker-compose/rsyslog.repo /etc/yum.repos.d/
+RUN yum install -y rsyslog-omhttp
+RUN echo '$IncludeConfig /etc/rsyslog.conf' >> /var/lib/awx/rsyslog/rsyslog.conf
+RUN chmod 0775 /var/lib/awx/rsyslog/rsyslog.conf
 
 ENV HOME /var/lib/awx
 ENV PATH="/usr/local/n/versions/node/10.15.0/bin:${PATH}"

From 996d7ce054905efe1e47f5e693616e1ec0724508 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Thu, 12 Mar 2020 13:43:43 -0400
Subject: [PATCH 08/37] Move supervisor and rsyslog sock files to their own
 dirs under /var/run

---
 awx/main/utils/external_logging.py            |  2 +-
 awx/settings/defaults.py                      |  2 +-
 .../roles/image_build/files/supervisor.conf   |  6 ++---
 .../kubernetes/templates/deployment.yml.j2    | 22 ++++++++++++-------
 .../templates/docker-compose.yml.j2           | 13 ++++++-----
 tools/docker-compose/supervisor.conf          |  2 +-
 6 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 1d373293db..4468634903 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -28,7 +28,7 @@ def reconfigure_rsyslog():
                 port = settings.LOG_AGGREGATOR_PORT
 
         parts.extend([
-            'input(type="imuxsock" Socket="/var/run/tower/sockets/rsyslog.sock" unlink="on")',
+            'input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")',
             'template(name="awx" type="string" string="%msg%")',
         ])
         if protocol.startswith('http'):
diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
index 0207f92b55..ab9709f50d 100644
--- a/awx/settings/defaults.py
+++ b/awx/settings/defaults.py
@@ -955,7 +955,7 @@ CHANNEL_LAYERS = {
 }
 
 # Logging configuration.
-LOGGING_SOCK = '/var/run/tower/sockets/rsyslog.sock'
+LOGGING_SOCK = '/var/run/rsyslog/rsyslog.sock'
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
diff --git a/installer/roles/image_build/files/supervisor.conf b/installer/roles/image_build/files/supervisor.conf
index bd49186170..74bd828326 100644
--- a/installer/roles/image_build/files/supervisor.conf
+++ b/installer/roles/image_build/files/supervisor.conf
@@ -47,7 +47,7 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
 [program:awx-rsyslogd]
-command = rsyslogd -n -i /var/run/tower/sockets/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
+command = rsyslogd -n -i /var/run/rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
 autostart = true
 autorestart = true
 stopwaitsecs = 1
@@ -74,10 +74,10 @@ events=TICK_60
 priority=0
 
 [unix_http_server]
-file=/var/run/tower/sockets/supervisor.web.sock
+file=/var/run/supervisor/supervisor.web.sock
 
 [supervisorctl]
-serverurl=unix:///var/run/tower/sockets/supervisor.web.sock ; use a unix:// URL  for a unix socket
+serverurl=unix:///var/run/supervisor/supervisor.web.sock ; use a unix:// URL  for a unix socket
 
 [rpcinterface:supervisor]
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2
index cdf414f28b..8e3c1f2a01 100644
--- a/installer/roles/kubernetes/templates/deployment.yml.j2
+++ b/installer/roles/kubernetes/templates/deployment.yml.j2
@@ -94,9 +94,11 @@ spec:
           ports:
             - containerPort: 8052
           volumeMounts:
-            - name: sockets
-              mountPath: "/var/run/tower/sockets/"
-            - name: rsyslog
+            - name: supervisor-socket
+              mountPath: "/var/run/supervisor"
+            - name: rsyslog-socket
+              mountPath: "/var/run/rsyslog"
+            - name: rsyslog-config
               mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
@@ -178,9 +180,11 @@ spec:
             - /usr/bin/launch_awx_task.sh
           imagePullPolicy: Always
           volumeMounts:
-            - name: sockets
-              mountPath: "/var/run/tower/sockets/"
-            - name: rsyslog
+            - name: supervisor-socket
+              mountPath: "/var/run/supervisor"
+            - name: rsyslog-socket
+              mountPath: "/var/run/rsyslog"
+            - name: rsyslog-config
               mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
@@ -323,9 +327,11 @@ spec:
 {{ affinity | to_nice_yaml(indent=2) | indent(width=8, indentfirst=True) }}
 {% endif %}
       volumes:
-        - name: sockets
+        - name: supervisor-socket
           emptyDir: {}
-        - name: rsyslog
+        - name: rsyslog-socket
+          emptyDir: {}
+        - name: rsyslog-config
           emptyDir: {}
 {% if ca_trust_dir is defined %}
         - name: {{ kubernetes_deployment_name }}-ca-trust-dir
diff --git a/installer/roles/local_docker/templates/docker-compose.yml.j2 b/installer/roles/local_docker/templates/docker-compose.yml.j2
index 3dfb449a73..da17d810a2 100644
--- a/installer/roles/local_docker/templates/docker-compose.yml.j2
+++ b/installer/roles/local_docker/templates/docker-compose.yml.j2
@@ -20,8 +20,8 @@ services:
     user: root
     restart: unless-stopped
     volumes:
-      - sockets:/var/run/tower/sockets/
-      - rsyslog:/var/lib/awx/rsyslog/
+      - rsyslog-socket:/var/run/rsyslog/
+      - rsyslog-config:/var/lib/awx/rsyslog/
       - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY"
       - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh"
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
@@ -77,8 +77,8 @@ services:
     user: root
     restart: unless-stopped
     volumes:
-      - sockets:/var/run/tower/sockets/
-      - rsyslog:/var/lib/awx/rsyslog/
+      - rsyslog-socket:/var/run/rsyslog/
+      - rsyslog-config:/var/lib/awx/rsyslog/
       - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY"
       - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh"
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
@@ -163,5 +163,6 @@ services:
       no_proxy: {{ no_proxy | default('') }}
   {% endif %}
 volumes:
-  sockets:
-  rsyslog:
+  supervisor-socket:
+  rsyslog-socket:
+  rsyslog-config:
diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf
index 826088c6ad..a28a230da1 100644
--- a/tools/docker-compose/supervisor.conf
+++ b/tools/docker-compose/supervisor.conf
@@ -72,7 +72,7 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
 [program:awx-rsyslogd]
-command = rsyslogd -n -i /var/run/tower/rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
+command = rsyslogd -n -i /var/run/rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
 autostart = true
 autorestart = true
 stopwaitsecs = 1

From d31c5282570e686fc6c9cd3dae9dd13f49e60216 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Wed, 11 Mar 2020 14:37:22 -0400
Subject: [PATCH 09/37] Fix Logging settings "Test" button functionality

---
 awx/api/templates/api/setting_logging_test.md |  1 +
 awx/conf/views.py                             | 42 +++++++++++++++++--
 awx/main/tasks.py                             |  2 +-
 awx/main/utils/reload.py                      |  2 -
 .../configuration-system.controller.js        |  2 +-
 tools/docker-compose/supervisor.conf          |  4 ++
 6 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/awx/api/templates/api/setting_logging_test.md b/awx/api/templates/api/setting_logging_test.md
index 149fac28ae..5b6c49dc57 100644
--- a/awx/api/templates/api/setting_logging_test.md
+++ b/awx/api/templates/api/setting_logging_test.md
@@ -1 +1,2 @@
 # Test Logging Configuration
+
diff --git a/awx/conf/views.py b/awx/conf/views.py
index 7a2a21a713..4b1f070d98 100644
--- a/awx/conf/views.py
+++ b/awx/conf/views.py
@@ -5,6 +5,9 @@
 import collections
 import logging
 import sys
+import socket
+import os
+from urllib.parse import urlparse
 
 # Django
 from django.conf import settings
@@ -12,7 +15,7 @@ from django.http import Http404
 from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
-from rest_framework.exceptions import PermissionDenied, ValidationError
+from rest_framework.exceptions import PermissionDenied
 from rest_framework.response import Response
 from rest_framework import serializers
 from rest_framework import status
@@ -161,8 +164,41 @@ class SettingLoggingTest(GenericAPIView):
     filter_backends = []
 
     def post(self, request, *args, **kwargs):
-        logging.getLogger('awx').info('AWX Connection Test')
-        return Response(status=status.HTTP_202_ACCEPTED)
+        # Send test message to configured logger based on db settings
+        logging.getLogger('awx').error('AWX Connection Test Message')
+        
+        hostname = getattr(settings, 'LOG_AGGREGATOR_HOST', None)
+        protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', None)
+        
+        # Check if host is reacheable
+        host = urlparse(hostname).netloc
+        response = os.system("ping -c 1 " + host)
+        if response != 0:
+            return Response({'error': 'The host is not available'}, status=status.HTTP_400_BAD_REQUEST)
+        
+        # Check to ensure port is open at host
+        if protocol in ['udp', 'tcp']:
+            port = getattr(settings, 'LOG_AGGREGATOR_PORT', None)
+            # Error if port is not set when using UDP/TCP
+            if not port:
+                return Response({'error': 'Port required for ' + protocol}, status=status.HTTP_400_BAD_REQUEST)
+        else:
+            return Response(status=status.HTTP_202_ACCEPTED)
+        
+        # Error if logging is not enabled
+        enabled = getattr(settings, 'LOG_AGGREGATOR_ENABLED', False)
+        if not enabled:
+            return Response({'error': 'Logging not enabled'}, status=status.HTTP_400_BAD_REQUEST)
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+        try:
+            s.settimeout(.5)
+            s.connect((hostname, int(port)))
+            s.shutdown(2)
+            s.close()
+            return Response(status=status.HTTP_202_ACCEPTED)
+        except Exception as e:
+            return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
 
 
 # Create view functions for all of the class-based views to simplify inclusion
diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index ebf29bbe97..c69a0c7f78 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -72,7 +72,7 @@ from awx.main.utils import (get_ssh_version, update_scm_url,
                             ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager,
                             get_awx_version)
 from awx.main.utils.ansible import read_ansible_config
-from awx.main.utils.common import get_ansible_version, _get_ansible_version, get_custom_venv_choices
+from awx.main.utils.common import _get_ansible_version, get_custom_venv_choices
 from awx.main.utils.external_logging import reconfigure_rsyslog
 from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja
 from awx.main.utils.reload import stop_local_services
diff --git a/awx/main/utils/reload.py b/awx/main/utils/reload.py
index 04868f06d0..9c71697516 100644
--- a/awx/main/utils/reload.py
+++ b/awx/main/utils/reload.py
@@ -6,8 +6,6 @@ import subprocess
 import logging
 import os
 
-# Django
-from django.conf import settings
 
 logger = logging.getLogger('awx.main.utils.reload')
 
diff --git a/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js b/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
index 4178b2f929..63fed820d2 100644
--- a/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
+++ b/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
@@ -215,7 +215,7 @@ export default [
                         });
                     })
                     .catch(({ data, status }) => {
-                        if (status === 500) {
+                        if (status === 400 || status == 500) {
                             ngToast.danger({
                                 content: '<i class="fa fa-exclamation-triangle Toast-successIcon"></i>' +
                                     i18n._('Log aggregator test failed. <br> Detail: ') + $filter('sanitize')(data.error),
diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf
index a28a230da1..0b7fd029f8 100644
--- a/tools/docker-compose/supervisor.conf
+++ b/tools/docker-compose/supervisor.conf
@@ -72,7 +72,11 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
 [program:awx-rsyslogd]
+<<<<<<< HEAD
 command = rsyslogd -n -i /var/run/rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
+=======
+command = rsyslogd -n -i /var/run/tower/sockets/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
+>>>>>>> 3a8bd7c40... Fix Logging settings "Test" button functionality
 autostart = true
 autorestart = true
 stopwaitsecs = 1

From fb047b1267e63d8788e44fd7f3de80559c4c1f47 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Thu, 12 Mar 2020 17:45:52 -0400
Subject: [PATCH 10/37] Add unit tests for reconfiguring rsyslog & for test
 endpoint

---
 awx/conf/views.py                             |  3 +-
 .../tests/functional/api/test_settings.py     |  5 +
 awx/main/tests/unit/api/test_logger_test.py   | 94 +++++++++++++++++++
 awx/main/utils/external_logging.py            | 14 ++-
 tools/docker-compose/supervisor.conf          |  4 -
 5 files changed, 110 insertions(+), 10 deletions(-)
 create mode 100644 awx/main/tests/unit/api/test_logger_test.py

diff --git a/awx/conf/views.py b/awx/conf/views.py
index 4b1f070d98..1f82009291 100644
--- a/awx/conf/views.py
+++ b/awx/conf/views.py
@@ -8,6 +8,7 @@ import sys
 import socket
 import os
 from urllib.parse import urlparse
+from socket import SHUT_RDWR
 
 # Django
 from django.conf import settings
@@ -194,7 +195,7 @@ class SettingLoggingTest(GenericAPIView):
         try:
             s.settimeout(.5)
             s.connect((hostname, int(port)))
-            s.shutdown(2)
+            s.shutdown(SHUT_RDWR)
             s.close()
             return Response(status=status.HTTP_202_ACCEPTED)
         except Exception as e:
diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py
index 5c6cb16022..9bcd813aad 100644
--- a/awx/main/tests/functional/api/test_settings.py
+++ b/awx/main/tests/functional/api/test_settings.py
@@ -292,6 +292,11 @@ def test_logging_aggregrator_connection_test_with_masked_password(mocker, patch,
         create_settings = kwargs['custom_settings']
         assert getattr(create_settings, 'LOG_AGGREGATOR_PASSWORD') == 'password123'
 
+    # Update these tests ^^
+    # Test the `/api/v2/settings/logging/test` functionality
+    def test_logging_test():
+        pass
+
 
 @pytest.mark.django_db
 def test_logging_aggregrator_connection_test_invalid(mocker, get, post, admin):
diff --git a/awx/main/tests/unit/api/test_logger_test.py b/awx/main/tests/unit/api/test_logger_test.py
new file mode 100644
index 0000000000..616b2e4c25
--- /dev/null
+++ b/awx/main/tests/unit/api/test_logger_test.py
@@ -0,0 +1,94 @@
+import pytest
+
+from awx.main.utils.external_logging import construct_rsyslog_conf_template
+from awx.conf import settings_registry
+
+
+'''
+# Example User Data
+data_logstash = {
+    "LOG_AGGREGATOR_TYPE": "logstash",
+    "LOG_AGGREGATOR_HOST": "localhost",
+    "LOG_AGGREGATOR_PORT": 8080,
+    "LOG_AGGREGATOR_PROTOCOL": "tcp",
+    "LOG_AGGREGATOR_USERNAME": "logger",
+    "LOG_AGGREGATOR_PASSWORD": "mcstash"
+}
+
+data_netcat = {
+    "LOG_AGGREGATOR_TYPE": "other",
+    "LOG_AGGREGATOR_HOST": "localhost",
+    "LOG_AGGREGATOR_PORT": 9000,
+    "LOG_AGGREGATOR_PROTOCOL": "udp",
+}
+
+data_loggly = {
+    "LOG_AGGREGATOR_TYPE": "loggly",
+    "LOG_AGGREGATOR_HOST": "http://logs-01.loggly.com/inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/",
+    "LOG_AGGREGATOR_PORT": 8080,
+    "LOG_AGGREGATOR_PROTOCOL": "https"
+}
+'''
+
+
+# Test reconfigure logging settings function
+# name this whatever you want
+@pytest.mark.parametrize(
+    'enabled, type, host, port, protocol, expected_config', [
+        (True, 'loggly', 'http://logs-01.loggly.com/inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/', None, 'https', 
+            '$IncludeConfig /etc/rsyslog.conf\ninput(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")\ntemplate(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")\naction(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")'),
+        (True, 'other', 'localhost', 9000, 'udp', 
+            '$IncludeConfig /etc/rsyslog.conf\ninput(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")\ntemplate(name="awx" type="string" string="%msg%")\naction(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" template="awx")'),
+        (True, 'other', 'localhost', 9000, 'tcp', 
+            '$IncludeConfig /etc/rsyslog.conf\ninput(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")\ntemplate(name="awx" type="string" string="%msg%")\naction(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" template="awx")'),
+        (False, 'loggly', 'http://logs-01.loggly.com/inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/', 8080, 'https', 
+            '$IncludeConfig /etc/rsyslog.conf'),
+        (True, 'splunk', 'https://yoursplunk:8088/services/collector/event', None, None,
+            '''$IncludeConfig /etc/rsyslog.conf
+input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")
+module(load="omhttp")
+action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'''),
+        (True, 'splunk', 'https://yoursplunk/services/collector/event', 8088, None,
+            '''$IncludeConfig /etc/rsyslog.conf
+input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")
+module(load="omhttp")
+action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'''),
+        (True, 'splunk', 'https://yoursplunk/services/collector/event', 8088, 'https',
+            '''$IncludeConfig /etc/rsyslog.conf
+input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")
+module(load="omhttp")
+action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'''),
+    ]
+)
+def test_rsyslog_conf_template(enabled, type, host, port, protocol, expected_config):
+    
+    # Mock settings object
+    class MockSettings:
+        pass
+    mock_settings = MockSettings()
+    
+    # Pre-populate settings obj with defaults
+    for key in settings_registry.get_registered_settings(category_slug='logging'):
+        value = settings_registry.get_setting_field(key).get_default()
+        setattr(mock_settings, key, value)
+    
+    
+    # Set test settings
+    setattr(mock_settings, 'LOGGING_SOCK', '/var/run/rsyslog/rsyslog.sock')
+    setattr(mock_settings, 'LOG_AGGREGATOR_ENABLED', enabled)
+    setattr(mock_settings, 'LOG_AGGREGATOR_TYPE', type)
+    setattr(mock_settings, 'LOG_AGGREGATOR_HOST', host)
+    if port:
+        setattr(mock_settings, 'LOG_AGGREGATOR_PORT', port)
+    if protocol:
+        setattr(mock_settings, 'LOG_AGGREGATOR_PROTOCOL', protocol)
+
+    # create rsyslog conf template 
+    tmpl = construct_rsyslog_conf_template(mock_settings)
+    
+    # check validity of created template
+    assert tmpl in expected_config
+
diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 4468634903..e54c50cbf9 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -5,14 +5,14 @@ from django.conf import settings
 from awx.main.utils.reload import supervisor_service_command
 
 
-def reconfigure_rsyslog():
+def construct_rsyslog_conf_template(settings=settings):
     tmpl = ''
     parts = ['$IncludeConfig /etc/rsyslog.conf']
     if settings.LOG_AGGREGATOR_ENABLED:
         host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
         port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
         protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '')
-
+        import pdb; pdb.set_trace()
         if protocol.startswith('http'):
             scheme = 'https'
             # urlparse requires '//' to be provided if scheme is not specified
@@ -23,12 +23,13 @@ def reconfigure_rsyslog():
 
             host = parsed.hostname
             try:
-                port = parsed.port
+                if parsed.port:
+                    port = parsed.port
             except ValueError:
                 port = settings.LOG_AGGREGATOR_PORT
 
         parts.extend([
-            'input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")',
+            'input(type="imuxsock" Socket="' + settings.LOGGING_SOCK + '" unlink="on")',
             'template(name="awx" type="string" string="%msg%")',
         ])
         if protocol.startswith('http'):
@@ -63,8 +64,11 @@ def reconfigure_rsyslog():
             parts.append(
                 f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
             )
-
     tmpl = '\n'.join(parts)
+    return tmpl
+    
+def reconfigure_rsyslog():
+    tmpl = get_rsyslog_conf_template()
     with open('/var/lib/awx/rsyslog/rsyslog.conf', 'w') as f:
         f.write(tmpl + '\n')
     supervisor_service_command(command='restart', service='awx-rsyslogd')
diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf
index 0b7fd029f8..a28a230da1 100644
--- a/tools/docker-compose/supervisor.conf
+++ b/tools/docker-compose/supervisor.conf
@@ -72,11 +72,7 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
 [program:awx-rsyslogd]
-<<<<<<< HEAD
 command = rsyslogd -n -i /var/run/rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
-=======
-command = rsyslogd -n -i /var/run/tower/sockets/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
->>>>>>> 3a8bd7c40... Fix Logging settings "Test" button functionality
 autostart = true
 autorestart = true
 stopwaitsecs = 1

From eb12f45e8e58c0506f140f238f4053b4f60b2ed8 Mon Sep 17 00:00:00 2001
From: John Mitchell <jmitchel@Johns-MacBook-Pro-46.local>
Date: Fri, 13 Mar 2020 15:47:58 -0400
Subject: [PATCH 11/37] add ngToast disable on timeout for log agg
 notifications, and disable test button until active test completes.

---
 .../forms/system-form/configuration-system.controller.js   | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js b/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
index 63fed820d2..520a0d5516 100644
--- a/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
+++ b/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
@@ -205,18 +205,25 @@ export default [
 
         $scope.$parent.vm.testLogging = function () {
             if (!$scope.$parent.vm.disableTestButton) {
+                $scope.$parent.vm.disableTestButton = true;
                 Rest.setUrl("/api/v2/settings/logging/test/");
                 Rest.post({})
                     .then(() => {
+                        $scope.$parent.vm.disableTestButton = false;
                         ngToast.success({
+                            dismissButton: false,
+                            dismissOnTimeout: true,
                             content: `<i class="fa fa-check-circle
                                 Toast-successIcon"></i>` +
                                 i18n._('Log aggregator test sent successfully.')
                         });
                     })
                     .catch(({ data, status }) => {
+                        $scope.$parent.vm.disableTestButton = false;
                         if (status === 400 || status == 500) {
                             ngToast.danger({
+                                dismissButton: false,
+                                dismissOnTimeout: true,
                                 content: '<i class="fa fa-exclamation-triangle Toast-successIcon"></i>' +
                                     i18n._('Log aggregator test failed. <br> Detail: ') + $filter('sanitize')(data.error),
                                 additionalClasses: "LogAggregator-failedNotification"

From 7fd79b8e546acef08f152ba278dd601f4af6dbd4 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Fri, 13 Mar 2020 17:31:04 -0400
Subject: [PATCH 12/37] Remove unneeded logging sock variable

---
 awx/main/utils/external_logging.py | 4 ++--
 awx/main/utils/handlers.py         | 2 +-
 awx/settings/defaults.py           | 3 +--
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index e54c50cbf9..49f74e4146 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -29,7 +29,7 @@ def construct_rsyslog_conf_template(settings=settings):
                 port = settings.LOG_AGGREGATOR_PORT
 
         parts.extend([
-            'input(type="imuxsock" Socket="' + settings.LOGGING_SOCK + '" unlink="on")',
+            'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger'] + '" unlink="on")',
             'template(name="awx" type="string" string="%msg%")',
         ])
         if protocol.startswith('http'):
@@ -68,7 +68,7 @@ def construct_rsyslog_conf_template(settings=settings):
     return tmpl
     
 def reconfigure_rsyslog():
-    tmpl = get_rsyslog_conf_template()
+    tmpl = construct_rsyslog_conf_template()
     with open('/var/lib/awx/rsyslog/rsyslog.conf', 'w') as f:
         f.write(tmpl + '\n')
     supervisor_service_command(command='restart', service='awx-rsyslogd')
diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py
index 26b4282fd1..7f393711d0 100644
--- a/awx/main/utils/handlers.py
+++ b/awx/main/utils/handlers.py
@@ -12,7 +12,7 @@ from django.conf import settings
 class RSysLogHandler(logging.handlers.SysLogHandler):
 
     def emit(self, msg):
-        if not os.path.exists(settings.LOGGING_SOCK):
+        if not os.path.exists(settings.LOGGING['handlers']['external_logger']):
             return
         return super(RSysLogHandler, self).emit(msg)
 
diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
index ab9709f50d..e024c4191d 100644
--- a/awx/settings/defaults.py
+++ b/awx/settings/defaults.py
@@ -955,7 +955,6 @@ CHANNEL_LAYERS = {
 }
 
 # Logging configuration.
-LOGGING_SOCK = '/var/run/rsyslog/rsyslog.sock'
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -1014,7 +1013,7 @@ LOGGING = {
         'external_logger': {
             'class': 'awx.main.utils.handlers.RSysLogHandler',
             'formatter': 'json',
-            'address': LOGGING_SOCK,
+            'address': '/var/run/rsyslog/rsyslog.sock',
             'filters': ['external_log_enabled', 'dynamic_level_filter'],
         },
         'tower_warnings': {

From d3505515478b4a940d03d73bb08741483c813790 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Mon, 16 Mar 2020 12:42:08 -0400
Subject: [PATCH 13/37] Tweaks to Test Button logic and cleans up flake8 and
 test failures

---
 awx/conf/tests/functional/test_api.py         |  14 --
 awx/conf/views.py                             |  20 +--
 .../tests/functional/api/test_settings.py     | 140 ++++++++++--------
 .../{test_logger_test.py => test_logger.py}   |  47 +++---
 awx/main/tests/unit/utils/test_reload.py      |   2 +-
 awx/main/utils/external_logging.py            |   6 +-
 awx/main/utils/handlers.py                    |   2 +-
 .../configuration-system.controller.js        |   4 +-
 docs/logging_integration.md                   |   3 +-
 requirements/README.md                        |   2 +-
 requirements/requirements.txt                 |   3 +-
 tools/docker-compose/supervisor.conf          |   2 +-
 12 files changed, 119 insertions(+), 126 deletions(-)
 rename awx/main/tests/unit/api/{test_logger_test.py => test_logger.py} (62%)

diff --git a/awx/conf/tests/functional/test_api.py b/awx/conf/tests/functional/test_api.py
index d7bb06a1bf..869627878a 100644
--- a/awx/conf/tests/functional/test_api.py
+++ b/awx/conf/tests/functional/test_api.py
@@ -325,17 +325,3 @@ def test_setting_singleton_delete_no_read_only_fields(api_request, dummy_setting
         )
         assert response.data['FOO_BAR'] == 23
 
-
-@pytest.mark.django_db
-def test_setting_logging_test(api_request):
-    with mock.patch('awx.conf.views.AWXProxyHandler.perform_test') as mock_func:
-        api_request(
-            'post',
-            reverse('api:setting_logging_test'),
-            data={'LOG_AGGREGATOR_HOST': 'http://foobar', 'LOG_AGGREGATOR_TYPE': 'logstash'}
-        )
-        call = mock_func.call_args_list[0]
-        args, kwargs = call
-        given_settings = kwargs['custom_settings']
-        assert given_settings.LOG_AGGREGATOR_HOST == 'http://foobar'
-        assert given_settings.LOG_AGGREGATOR_TYPE == 'logstash'
diff --git a/awx/conf/views.py b/awx/conf/views.py
index 1f82009291..0bd93bf7f2 100644
--- a/awx/conf/views.py
+++ b/awx/conf/views.py
@@ -6,8 +6,6 @@ import collections
 import logging
 import sys
 import socket
-import os
-from urllib.parse import urlparse
 from socket import SHUT_RDWR
 
 # Django
@@ -165,18 +163,17 @@ class SettingLoggingTest(GenericAPIView):
     filter_backends = []
 
     def post(self, request, *args, **kwargs):
+        # Error if logging is not enabled
+        enabled = getattr(settings, 'LOG_AGGREGATOR_ENABLED', False)
+        if not enabled:
+            return Response({'error': 'Logging not enabled'}, status=status.HTTP_400_BAD_REQUEST)
+        
         # Send test message to configured logger based on db settings
         logging.getLogger('awx').error('AWX Connection Test Message')
         
         hostname = getattr(settings, 'LOG_AGGREGATOR_HOST', None)
         protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', None)
         
-        # Check if host is reacheable
-        host = urlparse(hostname).netloc
-        response = os.system("ping -c 1 " + host)
-        if response != 0:
-            return Response({'error': 'The host is not available'}, status=status.HTTP_400_BAD_REQUEST)
-        
         # Check to ensure port is open at host
         if protocol in ['udp', 'tcp']:
             port = getattr(settings, 'LOG_AGGREGATOR_PORT', None)
@@ -184,14 +181,9 @@ class SettingLoggingTest(GenericAPIView):
             if not port:
                 return Response({'error': 'Port required for ' + protocol}, status=status.HTTP_400_BAD_REQUEST)
         else:
+            # if http/https by this point, domain is reacheable
             return Response(status=status.HTTP_202_ACCEPTED)
-        
-        # Error if logging is not enabled
-        enabled = getattr(settings, 'LOG_AGGREGATOR_ENABLED', False)
-        if not enabled:
-            return Response({'error': 'Logging not enabled'}, status=status.HTTP_400_BAD_REQUEST)
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-
         try:
             s.settimeout(.5)
             s.connect((hostname, int(port)))
diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py
index 9bcd813aad..262befcc19 100644
--- a/awx/main/tests/functional/api/test_settings.py
+++ b/awx/main/tests/functional/api/test_settings.py
@@ -5,17 +5,13 @@
 # Python
 import pytest
 import os
-import time
 
 from django.conf import settings
 
-# Mock
-from unittest import mock
-
 # AWX
 from awx.api.versioning import reverse
 from awx.conf.models import Setting
-from awx.main.utils.handlers import AWXProxyHandler, LoggingConnectivityException
+from awx.conf.registry import settings_registry
 
 
 TEST_GIF_LOGO = 'data:image/gif;base64,R0lGODlhIQAjAPIAAP//////AP8AAMzMAJmZADNmAAAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQJCgAHACwAAAAAIQAjAAADo3i63P4wykmrvTjrzZsxXfR94WMQBFh6RECuixHMLyzPQ13ewZCvow9OpzEAjIBj79cJJmU+FceIVEZ3QRozxBttmyOBwPBtisdX4Bha3oxmS+llFIPHQXQKkiSEXz9PeklHBzx3hYNyEHt4fmmAhHp8Nz45KgV5FgWFOFEGmwWbGqEfniChohmoQZ+oqRiZDZhEgk81I4mwg4EKVbxzrDHBEAkAIfkECQoABwAsAAAAACEAIwAAA6V4utz+MMpJq724GpP15p1kEAQYQmOwnWjgrmxjuMEAx8rsDjZ+fJvdLWQAFAHGWo8FRM54JqIRmYTigDrDMqZTbbbMj0CgjTLHZKvPQH6CTx+a2vKR0XbbOsoZ7SphG057gjl+c0dGgzeGNiaBiSgbBQUHBV08NpOVlkMSk0FKjZuURHiiOJxQnSGfQJuoEKREejK0dFRGjoiQt7iOuLx0rgxYEQkAIfkECQoABwAsAAAAACEAIwAAA7h4utxnxslJDSGR6nrz/owxYB64QUEwlGaVqlB7vrAJscsd3Lhy+wBArGEICo3DUFH4QDqK0GMy51xOgcGlEAfJ+iAFie62chR+jYKaSAuQGOqwJp7jGQRDuol+F/jxZWsyCmoQfwYwgoM5Oyg1i2w0A2WQIW2TPYOIkleQmy+UlYygoaIPnJmapKmqKiusMmSdpjxypnALtrcHioq3ury7hGm3dnVosVpMWFmwREZbddDOSsjVswcJACH5BAkKAAcALAAAAAAhACMAAAOxeLrc/jDKSZUxNS9DCNYV54HURQwfGRlDEFwqdLVuGjOsW9/Odb0wnsUAKBKNwsMFQGwyNUHckVl8bqI4o43lA26PNkv1S9DtNuOeVirw+aTI3qWAQwnud1vhLSnQLS0GeFF+GoVKNF0fh4Z+LDQ6Bn5/MTNmL0mAl2E3j2aclTmRmYCQoKEDiaRDKFhJez6UmbKyQowHtzy1uEl8DLCnEktrQ2PBD1NxSlXKIW5hz6cJACH5BAkKAAcALAAAAAAhACMAAAOkeLrc/jDKSau9OOvNlTFd9H3hYxAEWDJfkK5LGwTq+g0zDR/GgM+10A04Cm56OANgqTRmkDTmSOiLMgFOTM9AnFJHuexzYBAIijZf2SweJ8ttbbXLmd5+wBiJosSCoGF/fXEeS1g8gHl9hxODKkh4gkwVIwUekESIhA4FlgV3PyCWG52WI2oGnR2lnUWpqhqVEF4Xi7QjhpsshpOFvLosrnpoEAkAIfkECQoABwAsAAAAACEAIwAAA6l4utz+MMpJq71YGpPr3t1kEAQXQltQnk8aBCa7bMMLy4wx1G8s072PL6SrGQDI4zBThCU/v50zCVhidIYgNPqxWZkDg0AgxB2K4vEXbBSvr1JtZ3uOext0x7FqovF6OXtfe1UzdjAxhINPM013ChtJER8FBQeVRX8GlpggFZWWfjwblTiigGZnfqRmpUKbljKxDrNMeY2eF4R8jUiSur6/Z8GFV2WBtwwJACH5BAkKAAcALAAAAAAhACMAAAO6eLrcZi3KyQwhkGpq8f6ONWQgaAxB8JTfg6YkO50pzD5xhaurhCsGAKCnEw6NucNDCAkyI8ugdAhFKpnJJdMaeiofBejowUseCr9GYa0j1GyMdVgjBxoEuPSZXWKf7gKBeHtzMms0gHgGfDIVLztmjScvNZEyk28qjT40b5aXlHCbDgOhnzedoqOOlKeopaqrCy56sgtotbYKhYW6e7e9tsHBssO6eSTIm1peV0iuFUZDyU7NJnmcuQsJACH5BAkKAAcALAAAAAAhACMAAAOteLrc/jDKSZsxNS9DCNYV54Hh4H0kdAXBgKaOwbYX/Miza1vrVe8KA2AoJL5gwiQgeZz4GMXlcHl8xozQ3kW3KTajL9zsBJ1+sV2fQfALem+XAlRApxu4ioI1UpC76zJ4fRqDBzI+LFyFhH1iiS59fkgziW07jjRAG5QDeECOLk2Tj6KjnZafW6hAej6Smgevr6yysza2tiCuMasUF2Yov2gZUUQbU8YaaqjLpQkAOw==' # NOQA
@@ -237,78 +233,94 @@ def test_ui_settings(get, put, patch, delete, admin):
 
 
 @pytest.mark.django_db
-def test_logging_aggregrator_connection_test_requires_superuser(get, post, alice):
+def test_logging_aggregator_connection_test_requires_superuser(post, alice):
     url = reverse('api:setting_logging_test')
     post(url, {}, user=alice, expect=403)
 
 
-@pytest.mark.parametrize('key', [
-    'LOG_AGGREGATOR_TYPE',
-    'LOG_AGGREGATOR_HOST',
-])
 @pytest.mark.django_db
-def test_logging_aggregrator_connection_test_bad_request(get, post, admin, key):
+def test_logging_aggregator_connection_test_not_enabled(post, admin):
     url = reverse('api:setting_logging_test')
     resp = post(url, {}, user=admin, expect=400)
-    assert 'This field is required.' in resp.data.get(key, [])
+    assert 'Logging not enabled' in resp.data.get('error')
 
 
-@pytest.mark.django_db
-def test_logging_aggregrator_connection_test_valid(mocker, get, post, admin):
-    with mock.patch.object(AWXProxyHandler, 'perform_test') as perform_test:
-        url = reverse('api:setting_logging_test')
-        user_data = {
-            'LOG_AGGREGATOR_TYPE': 'logstash',
-            'LOG_AGGREGATOR_HOST': 'localhost',
-            'LOG_AGGREGATOR_PORT': 8080,
-            'LOG_AGGREGATOR_USERNAME': 'logger',
-            'LOG_AGGREGATOR_PASSWORD': 'mcstash'
-        }
-        post(url, user_data, user=admin, expect=200)
-        args, kwargs = perform_test.call_args_list[0]
-        create_settings = kwargs['custom_settings']
-        for k, v in user_data.items():
-            assert hasattr(create_settings, k)
-            assert getattr(create_settings, k) == v
-
-
-@pytest.mark.django_db
-def test_logging_aggregrator_connection_test_with_masked_password(mocker, patch, post, admin):
-    url = reverse('api:setting_singleton_detail', kwargs={'category_slug': 'logging'})
-    patch(url, user=admin, data={'LOG_AGGREGATOR_PASSWORD': 'password123'}, expect=200)
-    time.sleep(1)  # log settings are cached slightly
-
-    with mock.patch.object(AWXProxyHandler, 'perform_test') as perform_test:
-        url = reverse('api:setting_logging_test')
-        user_data = {
-            'LOG_AGGREGATOR_TYPE': 'logstash',
-            'LOG_AGGREGATOR_HOST': 'localhost',
-            'LOG_AGGREGATOR_PORT': 8080,
-            'LOG_AGGREGATOR_USERNAME': 'logger',
-            'LOG_AGGREGATOR_PASSWORD': '$encrypted$'
-        }
-        post(url, user_data, user=admin, expect=200)
-        args, kwargs = perform_test.call_args_list[0]
-        create_settings = kwargs['custom_settings']
-        assert getattr(create_settings, 'LOG_AGGREGATOR_PASSWORD') == 'password123'
-
-    # Update these tests ^^
-    # Test the `/api/v2/settings/logging/test` functionality
-    def test_logging_test():
+def _mock_logging_defaults():
+    # Pre-populate settings obj with defaults
+    class MockSettings:
         pass
+    mock_settings_obj = MockSettings()
+    mock_settings_json = dict()
+    for key in settings_registry.get_registered_settings(category_slug='logging'):
+        value = settings_registry.get_setting_field(key).get_default()
+        setattr(mock_settings_obj, key, value)
+        mock_settings_json[key] = value
+    return mock_settings_obj, mock_settings_json
+
+
+
+@pytest.mark.parametrize('key, value, error', [
+    ['LOG_AGGREGATOR_TYPE', 'logstash', 'Cannot enable log aggregator without providing host.'],
+    ['LOG_AGGREGATOR_HOST', 'https://logstash', 'Cannot enable log aggregator without providing type.']
+])
+@pytest.mark.django_db
+def test_logging_aggregator_missing_settings(put, post, admin, key, value, error):
+    _, mock_settings = _mock_logging_defaults()
+    mock_settings['LOG_AGGREGATOR_ENABLED'] = True
+    mock_settings[key] = value
+    url = reverse('api:setting_singleton_detail', kwargs={'category_slug': 'logging'})
+    response = put(url, data=mock_settings, user=admin, expect=400)
+    assert error in str(response.data)
+
+
+@pytest.mark.parametrize('type, host, port, username, password',  [
+    ['logstash', 'localhost', 8080, 'logger', 'mcstash'],
+    ['loggly', 'http://logs-01.loggly.com/inputs/1fd38090-hash-h4a$h-8d80-t0k3n71/tag/http/', None, None, None],
+    ['splunk', 'https://yoursplunk:8088/services/collector/event', None, None, None],
+    ['other', '97.221.40.41', 9000, 'logger', 'mcstash'], 
+    ['sumologic', 'https://endpoint5.collection.us2.sumologic.com/receiver/v1/http/Zagnw_f9XGr_zZgd-_EPM0hb8_rUU7_RU8Q==',
+        None, None, None]
+])
+@pytest.mark.django_db
+def test_logging_aggregator_valid_settings(put, post, admin, type, host, port, username, password):
+    _, mock_settings = _mock_logging_defaults()
+    # type = 'splunk'
+    # host = 'https://yoursplunk:8088/services/collector/event'
+    mock_settings['LOG_AGGREGATOR_ENABLED'] = True
+    mock_settings['LOG_AGGREGATOR_TYPE'] = type
+    mock_settings['LOG_AGGREGATOR_HOST'] = host
+    if port:
+        mock_settings['LOG_AGGREGATOR_PORT'] = port
+    if username:
+        mock_settings['LOG_AGGREGATOR_USERNAME'] = username
+    if password:
+        mock_settings['LOG_AGGREGATOR_PASSWORD'] = password
+    url = reverse('api:setting_singleton_detail', kwargs={'category_slug': 'logging'})
+    response = put(url, data=mock_settings, user=admin, expect=200)
+    assert type in response.data.get('LOG_AGGREGATOR_TYPE')
+    assert host in response.data.get('LOG_AGGREGATOR_HOST')
+    if port:
+        assert port == response.data.get('LOG_AGGREGATOR_PORT')
+    if username:
+        assert username in response.data.get('LOG_AGGREGATOR_USERNAME')
+    if password:   # Note: password should be encrypted
+        assert '$encrypted$' in response.data.get('LOG_AGGREGATOR_PASSWORD')
 
 
 @pytest.mark.django_db
-def test_logging_aggregrator_connection_test_invalid(mocker, get, post, admin):
-    with mock.patch.object(AWXProxyHandler, 'perform_test') as perform_test:
-        perform_test.side_effect = LoggingConnectivityException('404: Not Found')
-        url = reverse('api:setting_logging_test')
-        resp = post(url, {
-            'LOG_AGGREGATOR_TYPE': 'logstash',
-            'LOG_AGGREGATOR_HOST': 'localhost',
-            'LOG_AGGREGATOR_PORT': 8080
-        }, user=admin, expect=500)
-        assert resp.data == {'error': '404: Not Found'}
+def test_logging_aggregator_connection_test_valid(put, post, admin):
+    _, mock_settings = _mock_logging_defaults()
+    type = 'other'
+    host = 'https://localhost'
+    mock_settings['LOG_AGGREGATOR_ENABLED'] = True
+    mock_settings['LOG_AGGREGATOR_TYPE'] = type
+    mock_settings['LOG_AGGREGATOR_HOST'] = host
+    # POST to save these mock settings
+    url = reverse('api:setting_singleton_detail', kwargs={'category_slug': 'logging'})
+    put(url, data=mock_settings, user=admin, expect=200)
+    # "Test" the logger
+    url = reverse('api:setting_logging_test')
+    post(url, {}, user=admin, expect=202)
 
 
 @pytest.mark.django_db
diff --git a/awx/main/tests/unit/api/test_logger_test.py b/awx/main/tests/unit/api/test_logger.py
similarity index 62%
rename from awx/main/tests/unit/api/test_logger_test.py
rename to awx/main/tests/unit/api/test_logger.py
index 616b2e4c25..d08362259b 100644
--- a/awx/main/tests/unit/api/test_logger_test.py
+++ b/awx/main/tests/unit/api/test_logger.py
@@ -1,8 +1,9 @@
 import pytest
 
-from awx.main.utils.external_logging import construct_rsyslog_conf_template
-from awx.conf import settings_registry
+from django.conf import settings
 
+from awx.main.utils.external_logging import construct_rsyslog_conf_template
+from awx.main.tests.functional.api.test_settings import _mock_logging_defaults
 
 '''
 # Example User Data
@@ -36,48 +37,53 @@ data_loggly = {
 @pytest.mark.parametrize(
     'enabled, type, host, port, protocol, expected_config', [
         (True, 'loggly', 'http://logs-01.loggly.com/inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/', None, 'https', 
-            '$IncludeConfig /etc/rsyslog.conf\ninput(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")\ntemplate(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")\naction(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")'),
+            '''$IncludeConfig /etc/rsyslog.conf\ninput(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")
+action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" ''' +
+            'errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")'),
         (True, 'other', 'localhost', 9000, 'udp', 
-            '$IncludeConfig /etc/rsyslog.conf\ninput(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")\ntemplate(name="awx" type="string" string="%msg%")\naction(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" template="awx")'),
+            '''$IncludeConfig /etc/rsyslog.conf
+input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")
+action(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" template="awx")'''),
         (True, 'other', 'localhost', 9000, 'tcp', 
-            '$IncludeConfig /etc/rsyslog.conf\ninput(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")\ntemplate(name="awx" type="string" string="%msg%")\naction(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" template="awx")'),
+            '''$IncludeConfig /etc/rsyslog.conf
+input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")
+action(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" template="awx")'''),
         (False, 'loggly', 'http://logs-01.loggly.com/inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/', 8080, 'https', 
-            '$IncludeConfig /etc/rsyslog.conf'),
+            '''$IncludeConfig /etc/rsyslog.conf'''),
         (True, 'splunk', 'https://yoursplunk:8088/services/collector/event', None, None,
             '''$IncludeConfig /etc/rsyslog.conf
 input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
 template(name="awx" type="string" string="%msg%")
 module(load="omhttp")
-action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'''),
+action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" ''' +
+            'errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'),
         (True, 'splunk', 'https://yoursplunk/services/collector/event', 8088, None,
             '''$IncludeConfig /etc/rsyslog.conf
 input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
 template(name="awx" type="string" string="%msg%")
 module(load="omhttp")
-action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'''),
+action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" ''' +
+            'errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'),
         (True, 'splunk', 'https://yoursplunk/services/collector/event', 8088, 'https',
             '''$IncludeConfig /etc/rsyslog.conf
 input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
 template(name="awx" type="string" string="%msg%")
 module(load="omhttp")
-action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'''),
+action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" ''' +
+            'errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'),
     ]
 )
 def test_rsyslog_conf_template(enabled, type, host, port, protocol, expected_config):
     
-    # Mock settings object
-    class MockSettings:
-        pass
-    mock_settings = MockSettings()
-    
-    # Pre-populate settings obj with defaults
-    for key in settings_registry.get_registered_settings(category_slug='logging'):
-        value = settings_registry.get_setting_field(key).get_default()
-        setattr(mock_settings, key, value)
-    
+    mock_settings, _ = _mock_logging_defaults()
     
     # Set test settings
-    setattr(mock_settings, 'LOGGING_SOCK', '/var/run/rsyslog/rsyslog.sock')
+    logging_defaults = getattr(settings, 'LOGGING')
+    setattr(mock_settings, 'LOGGING', logging_defaults)
+    setattr(mock_settings, 'LOGGING["handlers"]["external_logger"]["address"]', '/var/run/rsyslog/rsyslog.sock')
     setattr(mock_settings, 'LOG_AGGREGATOR_ENABLED', enabled)
     setattr(mock_settings, 'LOG_AGGREGATOR_TYPE', type)
     setattr(mock_settings, 'LOG_AGGREGATOR_HOST', host)
@@ -91,4 +97,3 @@ def test_rsyslog_conf_template(enabled, type, host, port, protocol, expected_con
     
     # check validity of created template
     assert tmpl in expected_config
-
diff --git a/awx/main/tests/unit/utils/test_reload.py b/awx/main/tests/unit/utils/test_reload.py
index 87c2689da8..525a90e6aa 100644
--- a/awx/main/tests/unit/utils/test_reload.py
+++ b/awx/main/tests/unit/utils/test_reload.py
@@ -8,7 +8,7 @@ def test_produce_supervisor_command(mocker):
     mock_process.communicate = communicate_mock
     Popen_mock = mocker.MagicMock(return_value=mock_process)
     with mocker.patch.object(reload.subprocess, 'Popen', Popen_mock):
-        reload._supervisor_service_command("restart")
+        reload.supervisor_service_command("restart")
         reload.subprocess.Popen.assert_called_once_with(
             ['supervisorctl', 'restart', 'tower-processes:*',],
             stderr=-1, stdin=-1, stdout=-1)
diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 49f74e4146..74325a3b12 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -12,7 +12,6 @@ def construct_rsyslog_conf_template(settings=settings):
         host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
         port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
         protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '')
-        import pdb; pdb.set_trace()
         if protocol.startswith('http'):
             scheme = 'https'
             # urlparse requires '//' to be provided if scheme is not specified
@@ -29,7 +28,7 @@ def construct_rsyslog_conf_template(settings=settings):
                 port = settings.LOG_AGGREGATOR_PORT
 
         parts.extend([
-            'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger'] + '" unlink="on")',
+            'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")',
             'template(name="awx" type="string" string="%msg%")',
         ])
         if protocol.startswith('http'):
@@ -66,7 +65,8 @@ def construct_rsyslog_conf_template(settings=settings):
             )
     tmpl = '\n'.join(parts)
     return tmpl
-    
+
+
 def reconfigure_rsyslog():
     tmpl = construct_rsyslog_conf_template()
     with open('/var/lib/awx/rsyslog/rsyslog.conf', 'w') as f:
diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py
index 7f393711d0..a883058df1 100644
--- a/awx/main/utils/handlers.py
+++ b/awx/main/utils/handlers.py
@@ -12,7 +12,7 @@ from django.conf import settings
 class RSysLogHandler(logging.handlers.SysLogHandler):
 
     def emit(self, msg):
-        if not os.path.exists(settings.LOGGING['handlers']['external_logger']):
+        if not os.path.exists(settings.LOGGING['handlers']['external_logger']['address']):
             return
         return super(RSysLogHandler, self).emit(msg)
 
diff --git a/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js b/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
index 520a0d5516..14b62e2ef9 100644
--- a/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
+++ b/awx/ui/client/src/configuration/forms/system-form/configuration-system.controller.js
@@ -199,7 +199,7 @@ export default [
                 $scope.$parent.vm.testTooltip = i18n._('Save and enable log aggregation before testing the log aggregator.');
             } else {
                 $scope.$parent.vm.disableTestButton = false;
-                $scope.$parent.vm.testTooltip = i18n._('Send a test response to the configured log aggregator.');
+                $scope.$parent.vm.testTooltip = i18n._('Send a test log message to the configured log aggregator.');
             }
         });
 
@@ -220,7 +220,7 @@ export default [
                     })
                     .catch(({ data, status }) => {
                         $scope.$parent.vm.disableTestButton = false;
-                        if (status === 400 || status == 500) {
+                        if (status === 400 || status === 500) {
                             ngToast.danger({
                                 dismissButton: false,
                                 dismissOnTimeout: true,
diff --git a/docs/logging_integration.md b/docs/logging_integration.md
index 7b0eba9eb9..f21dd9b2e6 100644
--- a/docs/logging_integration.md
+++ b/docs/logging_integration.md
@@ -213,7 +213,6 @@ with the traceback message.
 
 Log messages should be sent outside of the
 request-response cycle. For example, Loggly examples use
-`requests_futures.sessions.FuturesSession`, which does some
-threading work to fire the message without interfering with other
+rsyslog, which handles these messages without interfering with other
 operations. A timeout on the part of the log aggregation service should
 not cause Tower operations to hang.
diff --git a/requirements/README.md b/requirements/README.md
index 4ce399a7e9..af672ae20e 100644
--- a/requirements/README.md
+++ b/requirements/README.md
@@ -4,7 +4,7 @@ The `requirements.txt` and `requirements_ansible.txt` files are generated from `
 
 ## How To Use
 
-Commands should from inside `./requirements` directory of the awx repository.
+Commands should be run from inside the `./requirements` directory of the awx repository.
 
 Make sure you have `patch, awk, python3, python2, python3-venv, python2-virtualenv, pip2, pip3` installed. The development container image should have all these.
 
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 3136ca3171..00fbee267d 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -101,9 +101,8 @@ python3-saml==1.9.0       # via -r /awx_devel/requirements/requirements.in
 pytz==2019.3              # via django, irc, tempora, twilio
 pyyaml==5.3.1             # via -r /awx_devel/requirements/requirements.in, ansible-runner, djangorestframework-yaml, kubernetes
 redis==3.4.1              # via -r /awx_devel/requirements/requirements.in
-requests-futures==1.0.0   # via -r /awx_devel/requirements/requirements.in
 requests-oauthlib==1.3.0  # via kubernetes, msrest, social-auth-core
-requests==2.23.0          # via -r /awx_devel/requirements/requirements.in, adal, azure-keyvault, django-oauth-toolkit, kubernetes, msrest, requests-futures, requests-oauthlib, slackclient, social-auth-core, twilio
+requests==2.23.0          # via -r /awx_devel/requirements/requirements.in, adal, azure-keyvault, django-oauth-toolkit, kubernetes, msrest, requests-oauthlib, slackclient, social-auth-core, twilio
 rsa==4.0                  # via google-auth
 ruamel.yaml.clib==0.2.0   # via ruamel.yaml
 ruamel.yaml==0.16.10      # via openshift
diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf
index a28a230da1..75d8c05fce 100644
--- a/tools/docker-compose/supervisor.conf
+++ b/tools/docker-compose/supervisor.conf
@@ -84,7 +84,7 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
 [group:tower-processes]
-programs=awx-dispatcher,awx-receiver,awx-runworker,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast,awx-rsyslogd
+programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast,awx-rsyslogd
 priority=5
 
 [unix_http_server]

From 4d5507d344f0010db1937da109f2c25876f8b6c1 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Wed, 1 Apr 2020 02:02:04 -0400
Subject: [PATCH 14/37] Add default rsyslog.conf without including
 /etc/rsyslog.conf

---
 awx/main/utils/external_logging.py                  | 6 +++++-
 installer/roles/image_build/templates/Dockerfile.j2 | 4 ++++
 tools/docker-compose.yml                            | 1 +
 tools/docker-compose/Dockerfile                     | 3 +--
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 74325a3b12..672fb2fdec 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -7,7 +7,7 @@ from awx.main.utils.reload import supervisor_service_command
 
 def construct_rsyslog_conf_template(settings=settings):
     tmpl = ''
-    parts = ['$IncludeConfig /etc/rsyslog.conf']
+    parts = []
     if settings.LOG_AGGREGATOR_ENABLED:
         host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
         port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
@@ -63,6 +63,10 @@ def construct_rsyslog_conf_template(settings=settings):
             parts.append(
                 f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
             )
+    parts.extend([
+        '$WorkDirectory /var/lib/awx/rsyslog',
+        '$IncludeConfig /etc/rsyslog.d/*.conf'
+    ])
     tmpl = '\n'.join(parts)
     return tmpl
 
diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2
index 8b829e7976..b3d63e8157 100644
--- a/installer/roles/image_build/templates/Dockerfile.j2
+++ b/installer/roles/image_build/templates/Dockerfile.j2
@@ -115,6 +115,10 @@ RUN find /var/lib/awx -not -path '/var/lib/awx/venv*' | xargs chgrp root && \
     chmod +rx /usr/bin/config-watcher && \
     chmod u+s /usr/bin/bwrap # https://github.com/ansible/awx/issues/5224
 
+# Create default awx rsyslog.conf
+RUN echo -e '$WorkDirectory /var/lib/awx/rsyslog\n$IncludeConfig /etc/rsyslog.d/*.conf' >> /var/lib/awx/rsyslog/rsyslog.conf
+
+
 RUN ln -sf /dev/stdout /var/log/nginx/access.log && \
     ln -sf /dev/stderr /var/log/nginx/error.log
 
diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml
index 6a39ed5f13..4b4de2e596 100644
--- a/tools/docker-compose.yml
+++ b/tools/docker-compose.yml
@@ -9,6 +9,7 @@ services:
     hostname: awx
     command: launch_awx.sh
     environment:
+      PYTHONUNBUFFERED: 1
       CURRENT_UID:
       OS:
       SDB_HOST: 0.0.0.0
diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile
index 3cf89997b3..00c2167986 100644
--- a/tools/docker-compose/Dockerfile
+++ b/tools/docker-compose/Dockerfile
@@ -131,10 +131,9 @@ RUN for dir in /var/lib/awx/rsyslog /var/run/tower/rsyslog /var/log/tower/ /var/
 
 
 RUN chmod -R 0775 /var/lib/awx /var/lib/awx/rsyslog
-RUN chmod -R 0770 /var/lib/rsyslog                    # needed, or else: rsyslogd: imjournal: open on state file `/var/lib/rsyslog/imjournal.state' failed 
 ADD tools/docker-compose/rsyslog.repo /etc/yum.repos.d/
 RUN yum install -y rsyslog-omhttp
-RUN echo '$IncludeConfig /etc/rsyslog.conf' >> /var/lib/awx/rsyslog/rsyslog.conf
+RUN echo -e '$WorkDirectory /var/lib/awx/rsyslog\n$IncludeConfig /etc/rsyslog.d/*.conf' >> /var/lib/awx/rsyslog/rsyslog.conf
 RUN chmod 0775 /var/lib/awx/rsyslog/rsyslog.conf
 
 ENV HOME /var/lib/awx

From e7403407937b566488842a3239c09dcf8bac2b1b Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Mon, 6 Apr 2020 16:32:02 -0400
Subject: [PATCH 15/37] ConfigMap rsyslog conf files for k8

---
 installer/roles/kubernetes/tasks/main.yml       |  2 ++
 .../kubernetes/templates/deployment.yml.j2      | 17 ++++++++++++++---
 .../roles/kubernetes/templates/rsyslog.yml.j2   | 10 ++++++++++
 3 files changed, 26 insertions(+), 3 deletions(-)
 create mode 100644 installer/roles/kubernetes/templates/rsyslog.yml.j2

diff --git a/installer/roles/kubernetes/tasks/main.yml b/installer/roles/kubernetes/tasks/main.yml
index 3ff39968b4..84a21ffa16 100644
--- a/installer/roles/kubernetes/tasks/main.yml
+++ b/installer/roles/kubernetes/tasks/main.yml
@@ -214,6 +214,7 @@
     - 'deployment'
     - 'supervisor'
     - 'launch_awx'
+    - 'rsyslog'
   no_log: true
 
 - name: Apply Deployment
@@ -225,6 +226,7 @@
     - "{{ deployment }}"
     - "{{ supervisor }}"
     - "{{ launch_awx }}"
+    - "{{ rsyslog }}"
   no_log: true
 
 - name: Delete any existing management pod
diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2
index 8e3c1f2a01..080a2ae80a 100644
--- a/installer/roles/kubernetes/templates/deployment.yml.j2
+++ b/installer/roles/kubernetes/templates/deployment.yml.j2
@@ -98,7 +98,7 @@ spec:
               mountPath: "/var/run/supervisor"
             - name: rsyslog-socket
               mountPath: "/var/run/rsyslog"
-            - name: rsyslog-config
+            - name: rsyslog-dir
               mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
@@ -148,6 +148,11 @@ spec:
               subPath: supervisor_task.conf
               readOnly: true
 
+            - name: {{ kubernetes_deployment_name }}-rsyslog-config
+              mountPath: "/var/lib/awx/rsyslog/rsyslog.conf"
+              subPath: rsyslog.conf
+              readOnly: true
+
             - name: {{ kubernetes_deployment_name }}-secret-key
               mountPath: "/etc/tower/SECRET_KEY"
               subPath: SECRET_KEY
@@ -184,7 +189,7 @@ spec:
               mountPath: "/var/run/supervisor"
             - name: rsyslog-socket
               mountPath: "/var/run/rsyslog"
-            - name: rsyslog-config
+            - name: rsyslog-dir
               mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
@@ -331,7 +336,7 @@ spec:
           emptyDir: {}
         - name: rsyslog-socket
           emptyDir: {}
-        - name: rsyslog-config
+        - name: rsyslog-dir
           emptyDir: {}
 {% if ca_trust_dir is defined %}
         - name: {{ kubernetes_deployment_name }}-ca-trust-dir
@@ -409,6 +414,12 @@ spec:
               - key: supervisor-task-config
                 path: 'supervisor_task.conf'
 
+        - name: {{ kubernetes_deployment_name }}-rsyslog-config
+          configMap:
+            name: {{ kubernetes_deployment_name }}-rsyslog-config
+            items:
+              - key: rsyslog-config
+                path: 'rsyslog.conf'
 
         - name: {{ kubernetes_deployment_name }}-secret-key
           secret:
diff --git a/installer/roles/kubernetes/templates/rsyslog.yml.j2 b/installer/roles/kubernetes/templates/rsyslog.yml.j2
new file mode 100644
index 0000000000..3055c498ea
--- /dev/null
+++ b/installer/roles/kubernetes/templates/rsyslog.yml.j2
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ kubernetes_deployment_name }}-rsyslog-config
+  namespace: {{ kubernetes_namespace }}
+data:
+  rsyslog-config: |
+    $WorkDirectory /var/lib/awx/rsyslog
+    $IncludeConfig /etc/rsyslog.d/*.conf
+    

From 470159b4d7c50fb67c7416e94fcb01a98c8e0465 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Tue, 7 Apr 2020 18:39:36 -0400
Subject: [PATCH 16/37] Enable innocuous but valid config for rsyslog if
 disabled

---
 awx/conf/views.py                  |  8 ++++++--
 awx/main/utils/external_logging.py | 16 +++++++++++-----
 tools/docker-compose/Dockerfile    |  2 +-
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/awx/conf/views.py b/awx/conf/views.py
index 0bd93bf7f2..ad1d62ad3a 100644
--- a/awx/conf/views.py
+++ b/awx/conf/views.py
@@ -166,7 +166,7 @@ class SettingLoggingTest(GenericAPIView):
         # Error if logging is not enabled
         enabled = getattr(settings, 'LOG_AGGREGATOR_ENABLED', False)
         if not enabled:
-            return Response({'error': 'Logging not enabled'}, status=status.HTTP_400_BAD_REQUEST)
+            return Response({'error': 'Logging not enabled'}, status=status.HTTP_409_CONFLICT)
         
         # Send test message to configured logger based on db settings
         logging.getLogger('awx').error('AWX Connection Test Message')
@@ -183,7 +183,11 @@ class SettingLoggingTest(GenericAPIView):
         else:
             # if http/https by this point, domain is reacheable
             return Response(status=status.HTTP_202_ACCEPTED)
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+        if protocol is 'udp':
+            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        else:
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         try:
             s.settimeout(.5)
             s.connect((hostname, int(port)))
diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 672fb2fdec..fd580d64cf 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -26,8 +26,10 @@ def construct_rsyslog_conf_template(settings=settings):
                     port = parsed.port
             except ValueError:
                 port = settings.LOG_AGGREGATOR_PORT
-
         parts.extend([
+            '$WorkDirectory /var/lib/awx/rsyslog',
+            '$IncludeConfig /etc/rsyslog.d/*.conf',
+            '$ModLoad imuxsock',
             'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")',
             'template(name="awx" type="string" string="%msg%")',
         ])
@@ -63,10 +65,14 @@ def construct_rsyslog_conf_template(settings=settings):
             parts.append(
                 f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
             )
-    parts.extend([
-        '$WorkDirectory /var/lib/awx/rsyslog',
-        '$IncludeConfig /etc/rsyslog.d/*.conf'
-    ])
+    else:
+        # If logging is disabled, add a valid config and discard all messages
+        parts = [
+            '$WorkDirectory /var/lib/awx/rsyslog',
+            '$IncludeConfig /etc/rsyslog.d/*.conf',
+            '*.* stop'
+            f'action(type="omfwd" target="localhost" port="9000" protocol="udp")'
+        ]
     tmpl = '\n'.join(parts)
     return tmpl
 
diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile
index 00c2167986..ef7b8ba57a 100644
--- a/tools/docker-compose/Dockerfile
+++ b/tools/docker-compose/Dockerfile
@@ -123,7 +123,7 @@ ADD tools/docker-compose/entrypoint.sh /
 ADD tools/scripts/awx-python /usr/bin/awx-python
 
 # Pre-create things that we need to write to
-RUN for dir in /var/lib/awx/rsyslog /var/run/tower/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
+RUN for dir in /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \

From 2329c1b7977227854428e726e478d6977aee7341 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Wed, 8 Apr 2020 00:50:20 -0400
Subject: [PATCH 17/37] Add rsyslog config to container from file for
 consistency

---
 awx/main/utils/external_logging.py                  | 8 --------
 installer/roles/image_build/files/rsyslog.conf      | 4 ++++
 installer/roles/image_build/templates/Dockerfile.j2 | 4 ++--
 installer/roles/kubernetes/templates/rsyslog.yml.j2 | 3 ++-
 tools/docker-compose/Dockerfile                     | 4 ++--
 tools/docker-compose/rsyslog.conf                   | 4 ++++
 6 files changed, 14 insertions(+), 13 deletions(-)
 create mode 100644 installer/roles/image_build/files/rsyslog.conf
 create mode 100644 tools/docker-compose/rsyslog.conf

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index fd580d64cf..3f37e79f11 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -65,14 +65,6 @@ def construct_rsyslog_conf_template(settings=settings):
             parts.append(
                 f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
             )
-    else:
-        # If logging is disabled, add a valid config and discard all messages
-        parts = [
-            '$WorkDirectory /var/lib/awx/rsyslog',
-            '$IncludeConfig /etc/rsyslog.d/*.conf',
-            '*.* stop'
-            f'action(type="omfwd" target="localhost" port="9000" protocol="udp")'
-        ]
     tmpl = '\n'.join(parts)
     return tmpl
 
diff --git a/installer/roles/image_build/files/rsyslog.conf b/installer/roles/image_build/files/rsyslog.conf
new file mode 100644
index 0000000000..a1fc3407b3
--- /dev/null
+++ b/installer/roles/image_build/files/rsyslog.conf
@@ -0,0 +1,4 @@
+$WorkDirectory /var/lib/awx/rsyslog
+$IncludeConfig /etc/rsyslog.d/*.conf
+*.* stop
+action(type="omfwd" target="localhost" port="9000" protocol="udp")
diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2
index b3d63e8157..3834ce3a76 100644
--- a/installer/roles/image_build/templates/Dockerfile.j2
+++ b/installer/roles/image_build/templates/Dockerfile.j2
@@ -101,7 +101,7 @@ ADD rsyslog.repo /etc/yum.repos.d/
 RUN yum install -y rsyslog-omhttp
 
 # Pre-create things that we need to write to
-RUN for dir in /home/awx /var/run/supervisor /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \
+RUN for dir in /home/awx /var/run/supervisor /var/lib/awx /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /var/run/nginx.pid; \
@@ -116,7 +116,7 @@ RUN find /var/lib/awx -not -path '/var/lib/awx/venv*' | xargs chgrp root && \
     chmod u+s /usr/bin/bwrap # https://github.com/ansible/awx/issues/5224
 
 # Create default awx rsyslog.conf
-RUN echo -e '$WorkDirectory /var/lib/awx/rsyslog\n$IncludeConfig /etc/rsyslog.d/*.conf' >> /var/lib/awx/rsyslog/rsyslog.conf
+ADD rsyslog.conf /var/lib/awx/rsyslog/rsyslog.conf
 
 
 RUN ln -sf /dev/stdout /var/log/nginx/access.log && \
diff --git a/installer/roles/kubernetes/templates/rsyslog.yml.j2 b/installer/roles/kubernetes/templates/rsyslog.yml.j2
index 3055c498ea..ba4705f08c 100644
--- a/installer/roles/kubernetes/templates/rsyslog.yml.j2
+++ b/installer/roles/kubernetes/templates/rsyslog.yml.j2
@@ -7,4 +7,5 @@ data:
   rsyslog-config: |
     $WorkDirectory /var/lib/awx/rsyslog
     $IncludeConfig /etc/rsyslog.d/*.conf
-    
+    *.* stop
+    action(type="omfwd" target="localhost" port="9000" protocol="udp")
diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile
index ef7b8ba57a..62536d7b18 100644
--- a/tools/docker-compose/Dockerfile
+++ b/tools/docker-compose/Dockerfile
@@ -123,7 +123,7 @@ ADD tools/docker-compose/entrypoint.sh /
 ADD tools/scripts/awx-python /usr/bin/awx-python
 
 # Pre-create things that we need to write to
-RUN for dir in /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
+RUN for dir in /var/lib/awx /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \
@@ -133,7 +133,7 @@ RUN for dir in /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower/ /var/lib/aw
 RUN chmod -R 0775 /var/lib/awx /var/lib/awx/rsyslog
 ADD tools/docker-compose/rsyslog.repo /etc/yum.repos.d/
 RUN yum install -y rsyslog-omhttp
-RUN echo -e '$WorkDirectory /var/lib/awx/rsyslog\n$IncludeConfig /etc/rsyslog.d/*.conf' >> /var/lib/awx/rsyslog/rsyslog.conf
+ADD tools/docker-compose/rsyslog.conf /var/lib/awx/rsyslog/rsyslog.conf
 RUN chmod 0775 /var/lib/awx/rsyslog/rsyslog.conf
 
 ENV HOME /var/lib/awx
diff --git a/tools/docker-compose/rsyslog.conf b/tools/docker-compose/rsyslog.conf
new file mode 100644
index 0000000000..a1fc3407b3
--- /dev/null
+++ b/tools/docker-compose/rsyslog.conf
@@ -0,0 +1,4 @@
+$WorkDirectory /var/lib/awx/rsyslog
+$IncludeConfig /etc/rsyslog.d/*.conf
+*.* stop
+action(type="omfwd" target="localhost" port="9000" protocol="udp")

From 70391f96ae1eeaf7b1eb10cf130c4add6a33fa60 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Wed, 8 Apr 2020 11:41:57 -0400
Subject: [PATCH 18/37] Revert rsyslog valid config to one that fails
 intentionally

---
 installer/roles/image_build/files/rsyslog.conf      | 3 ---
 installer/roles/kubernetes/templates/rsyslog.yml.j2 | 4 +---
 tools/docker-compose/rsyslog.conf                   | 3 ---
 3 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/installer/roles/image_build/files/rsyslog.conf b/installer/roles/image_build/files/rsyslog.conf
index a1fc3407b3..cc67dc4ebd 100644
--- a/installer/roles/image_build/files/rsyslog.conf
+++ b/installer/roles/image_build/files/rsyslog.conf
@@ -1,4 +1 @@
 $WorkDirectory /var/lib/awx/rsyslog
-$IncludeConfig /etc/rsyslog.d/*.conf
-*.* stop
-action(type="omfwd" target="localhost" port="9000" protocol="udp")
diff --git a/installer/roles/kubernetes/templates/rsyslog.yml.j2 b/installer/roles/kubernetes/templates/rsyslog.yml.j2
index ba4705f08c..4a3bbea968 100644
--- a/installer/roles/kubernetes/templates/rsyslog.yml.j2
+++ b/installer/roles/kubernetes/templates/rsyslog.yml.j2
@@ -6,6 +6,4 @@ metadata:
 data:
   rsyslog-config: |
     $WorkDirectory /var/lib/awx/rsyslog
-    $IncludeConfig /etc/rsyslog.d/*.conf
-    *.* stop
-    action(type="omfwd" target="localhost" port="9000" protocol="udp")
+    
diff --git a/tools/docker-compose/rsyslog.conf b/tools/docker-compose/rsyslog.conf
index a1fc3407b3..cc67dc4ebd 100644
--- a/tools/docker-compose/rsyslog.conf
+++ b/tools/docker-compose/rsyslog.conf
@@ -1,4 +1 @@
 $WorkDirectory /var/lib/awx/rsyslog
-$IncludeConfig /etc/rsyslog.d/*.conf
-*.* stop
-action(type="omfwd" target="localhost" port="9000" protocol="udp")

From ce82b87d9f622f4e13b3de788afc1b8d6780b35a Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Wed, 8 Apr 2020 13:38:35 -0400
Subject: [PATCH 19/37] rsyslog hardening (fixing a few weird things we
 noticed)

---
 awx/main/utils/external_logging.py | 2 ++
 awx/main/utils/formatters.py       | 2 +-
 awx/main/utils/handlers.py         | 2 ++
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 3f37e79f11..a8295ad399 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -26,9 +26,11 @@ def construct_rsyslog_conf_template(settings=settings):
                     port = parsed.port
             except ValueError:
                 port = settings.LOG_AGGREGATOR_PORT
+        max_bytes = settings.MAX_EVENT_RES_DATA
         parts.extend([
             '$WorkDirectory /var/lib/awx/rsyslog',
             '$IncludeConfig /etc/rsyslog.d/*.conf',
+            f'$MaxMessageSize {max_bytes}b',
             '$ModLoad imuxsock',
             'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")',
             'template(name="awx" type="string" string="%msg%")',
diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py
index 1c3146ee46..fb08034a5e 100644
--- a/awx/main/utils/formatters.py
+++ b/awx/main/utils/formatters.py
@@ -97,7 +97,7 @@ class LogstashFormatterBase(logging.Formatter):
 
     @classmethod
     def serialize(cls, message):
-        return bytes(json.dumps(message, cls=DjangoJSONEncoder), 'utf-8')
+        return ' ' + json.dumps(message, cls=DjangoJSONEncoder) + '\000'
 
 
 class LogstashFormatter(LogstashFormatterBase):
diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py
index a883058df1..f7c11cbc53 100644
--- a/awx/main/utils/handlers.py
+++ b/awx/main/utils/handlers.py
@@ -11,6 +11,8 @@ from django.conf import settings
 
 class RSysLogHandler(logging.handlers.SysLogHandler):
 
+    append_nul = False
+
     def emit(self, msg):
         if not os.path.exists(settings.LOGGING['handlers']['external_logger']['address']):
             return

From b942fde59a5536089d046abfe66d655d549f8c08 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Wed, 8 Apr 2020 14:37:21 -0400
Subject: [PATCH 20/37] Ensure log messages have valid json  - Fix messages
 getting contatenated at 8k  - Fix rsyslog cutting off the opening brace of
 log messages  - Make valid default conf and emit logs based on prescence of
 .sock and  settings

---
 awx/main/utils/external_logging.py            | 115 +++++++++---------
 awx/main/utils/handlers.py                    |   2 +
 .../roles/image_build/files/rsyslog.conf      |   5 +
 .../roles/image_build/templates/Dockerfile.j2 |   2 +-
 tools/docker-compose/Dockerfile               |   2 +-
 tools/docker-compose/rsyslog.conf             |   5 +
 6 files changed, 72 insertions(+), 59 deletions(-)

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index a8295ad399..e5e11d5156 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -8,65 +8,66 @@ from awx.main.utils.reload import supervisor_service_command
 def construct_rsyslog_conf_template(settings=settings):
     tmpl = ''
     parts = []
-    if settings.LOG_AGGREGATOR_ENABLED:
-        host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
-        port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
-        protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '')
-        if protocol.startswith('http'):
-            scheme = 'https'
-            # urlparse requires '//' to be provided if scheme is not specified
-            original_parsed = urlparse.urlsplit(host)
-            if (not original_parsed.scheme and not host.startswith('//')) or original_parsed.hostname is None:
-                host = '%s://%s' % (scheme, host) if scheme else '//%s' % host
-            parsed = urlparse.urlsplit(host)
+    host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
+    port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
+    protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '')
+    if protocol.startswith('http'):
+        scheme = 'https'
+        # urlparse requires '//' to be provided if scheme is not specified
+        original_parsed = urlparse.urlsplit(host)
+        if (not original_parsed.scheme and not host.startswith('//')) or original_parsed.hostname is None:
+            host = '%s://%s' % (scheme, host) if scheme else '//%s' % host
+        parsed = urlparse.urlsplit(host)
 
-            host = parsed.hostname
-            try:
-                if parsed.port:
-                    port = parsed.port
-            except ValueError:
-                port = settings.LOG_AGGREGATOR_PORT
-        max_bytes = settings.MAX_EVENT_RES_DATA
-        parts.extend([
-            '$WorkDirectory /var/lib/awx/rsyslog',
-            '$IncludeConfig /etc/rsyslog.d/*.conf',
-            f'$MaxMessageSize {max_bytes}b',
-            '$ModLoad imuxsock',
-            'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")',
-            'template(name="awx" type="string" string="%msg%")',
-        ])
-        if protocol.startswith('http'):
-            # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst
-            ssl = "on" if parsed.scheme == 'https' else "off"
-            skip_verify = "off" if settings.LOG_AGGREGATOR_VERIFY_CERT else "on"
-            if not port:
-                port = 443 if parsed.scheme == 'https' else 80
+        host = parsed.hostname
+        try:
+            if parsed.port:
+                port = parsed.port
+        except ValueError:
+            port = settings.LOG_AGGREGATOR_PORT
+    
+    max_bytes = settings.MAX_EVENT_RES_DATA
+    parts.extend([
+        '$WorkDirectory /var/lib/awx/rsyslog',
+        f'$MaxMessageSize {max_bytes}',
+        '$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf',
+        '$ModLoad imuxsock',
+        'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")',
+        'template(name="awx" type="string" string="%msg%")',
+    ])
+    if protocol.startswith('http'):
+        # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst
+        ssl = "on" if parsed.scheme == 'https' else "off"
+        skip_verify = "off" if settings.LOG_AGGREGATOR_VERIFY_CERT else "on"
+        if not port:
+            port = 443 if parsed.scheme == 'https' else 80
 
-            params = [
-                'type="omhttp"',
-                f'server="{host}"',
-                f'serverport="{port}"',
-                f'usehttps="{ssl}"',
-                f'skipverifyhost="{skip_verify}"',
-                'action.resumeRetryCount="-1"',
-                'template="awx"',
-                'errorfile="/var/log/tower/external.err"',
-                'healthchecktimeout="20000"',
-            ]
-            if parsed.path:
-                params.append(f'restpath="{parsed.path[1:]}"')
-            username = getattr(settings, 'LOG_AGGREGATOR_USERNAME', '')
-            password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '')
-            if username:
-                params.append(f'uid="{username}"')
-            if password:
-                params.append(f'pwd="{password}"')
-            params = ' '.join(params)
-            parts.extend(['module(load="omhttp")', f'action({params})'])
-        else:
-            parts.append(
-                f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
-            )
+        params = [
+            'type="omhttp"',
+            f'server="{host}"',
+            f'serverport="{port}"',
+            f'usehttps="{ssl}"',
+            f'skipverifyhost="{skip_verify}"',
+            'action.resumeRetryCount="-1"',
+            'template="awx"',
+            'errorfile="/var/log/tower/external.err"',
+            'healthchecktimeout="20000"',
+        ]
+        if parsed.path:
+            params.append(f'restpath="{parsed.path[1:]}"')
+        username = getattr(settings, 'LOG_AGGREGATOR_USERNAME', '')
+        password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '')
+        if username:
+            params.append(f'uid="{username}"')
+        if password:
+            params.append(f'pwd="{password}"')
+        params = ' '.join(params)
+        parts.extend(['module(load="omhttp")', f'action({params})'])
+    elif protocol and host and port:
+        parts.append(
+            f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
+        )
+    # parts.append('$IncludeConfig /var/lib/awx/rsyslog/*.conf')
     tmpl = '\n'.join(parts)
     return tmpl
 
diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py
index f7c11cbc53..ebf299a9f0 100644
--- a/awx/main/utils/handlers.py
+++ b/awx/main/utils/handlers.py
@@ -14,6 +14,8 @@ class RSysLogHandler(logging.handlers.SysLogHandler):
     append_nul = False
 
     def emit(self, msg):
+        if not settings.LOG_AGGREGATOR_ENABLED:
+            return
         if not os.path.exists(settings.LOGGING['handlers']['external_logger']['address']):
             return
         return super(RSysLogHandler, self).emit(msg)
diff --git a/installer/roles/image_build/files/rsyslog.conf b/installer/roles/image_build/files/rsyslog.conf
index cc67dc4ebd..c3a2d6727d 100644
--- a/installer/roles/image_build/files/rsyslog.conf
+++ b/installer/roles/image_build/files/rsyslog.conf
@@ -1 +1,6 @@
 $WorkDirectory /var/lib/awx/rsyslog
+$MaxMessageSize 700000
+$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf
+$ModLoad imuxsock
+input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")
diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2
index 3834ce3a76..50ec4686e1 100644
--- a/installer/roles/image_build/templates/Dockerfile.j2
+++ b/installer/roles/image_build/templates/Dockerfile.j2
@@ -101,7 +101,7 @@ ADD rsyslog.repo /etc/yum.repos.d/
 RUN yum install -y rsyslog-omhttp
 
 # Pre-create things that we need to write to
-RUN for dir in /home/awx /var/run/supervisor /var/lib/awx /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \
+RUN for dir in /home/awx /var/run/supervisor /var/lib/awx /var/lib/awx/rsyslog /var/lib/awx/rsyslog/conf.d /var/run/rsyslog /var/log/tower /var/log/nginx /var/lib/nginx; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /var/run/nginx.pid; \
diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile
index 62536d7b18..88d0af0bf4 100644
--- a/tools/docker-compose/Dockerfile
+++ b/tools/docker-compose/Dockerfile
@@ -123,7 +123,7 @@ ADD tools/docker-compose/entrypoint.sh /
 ADD tools/scripts/awx-python /usr/bin/awx-python
 
 # Pre-create things that we need to write to
-RUN for dir in /var/lib/awx /var/lib/awx/rsyslog /var/run/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
+RUN for dir in /var/lib/awx /var/lib/awx/rsyslog /var/lib/awx/rsyslog/conf.d /var/run/rsyslog /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \
     do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \
     \
     for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \
diff --git a/tools/docker-compose/rsyslog.conf b/tools/docker-compose/rsyslog.conf
index cc67dc4ebd..c3a2d6727d 100644
--- a/tools/docker-compose/rsyslog.conf
+++ b/tools/docker-compose/rsyslog.conf
@@ -1 +1,6 @@
 $WorkDirectory /var/lib/awx/rsyslog
+$MaxMessageSize 700000
+$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf
+$ModLoad imuxsock
+input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+template(name="awx" type="string" string="%msg%")

From 39648b4f0b3cac19616eda30c8f13170ae8f769a Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Wed, 8 Apr 2020 14:40:26 -0400
Subject: [PATCH 21/37] fix up a few test and lint errors related to external
 logging

---
 awx/conf/views.py                              | 2 +-
 awx/main/tests/functional/api/test_settings.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/awx/conf/views.py b/awx/conf/views.py
index ad1d62ad3a..580a0d2e52 100644
--- a/awx/conf/views.py
+++ b/awx/conf/views.py
@@ -184,7 +184,7 @@ class SettingLoggingTest(GenericAPIView):
             # if http/https by this point, domain is reacheable
             return Response(status=status.HTTP_202_ACCEPTED)
 
-        if protocol is 'udp':
+        if protocol == 'udp':
             s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         else:
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py
index 262befcc19..385bd32129 100644
--- a/awx/main/tests/functional/api/test_settings.py
+++ b/awx/main/tests/functional/api/test_settings.py
@@ -241,7 +241,7 @@ def test_logging_aggregator_connection_test_requires_superuser(post, alice):
 @pytest.mark.django_db
 def test_logging_aggregator_connection_test_not_enabled(post, admin):
     url = reverse('api:setting_logging_test')
-    resp = post(url, {}, user=admin, expect=400)
+    resp = post(url, {}, user=admin, expect=409)
     assert 'Logging not enabled' in resp.data.get('error')
 
 

From f7dac8e68d9196423c3500762f62f54ccb9361e2 Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Wed, 8 Apr 2020 14:43:03 -0400
Subject: [PATCH 22/37] more external logging unit test fixups

---
 .../tests/functional/api/test_settings.py     |   1 +
 awx/main/tests/unit/api/test_logger.py        | 139 +++++++++++++-----
 awx/main/utils/external_logging.py            |   1 -
 3 files changed, 101 insertions(+), 40 deletions(-)

diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py
index 385bd32129..67c9868649 100644
--- a/awx/main/tests/functional/api/test_settings.py
+++ b/awx/main/tests/functional/api/test_settings.py
@@ -255,6 +255,7 @@ def _mock_logging_defaults():
         value = settings_registry.get_setting_field(key).get_default()
         setattr(mock_settings_obj, key, value)
         mock_settings_json[key] = value
+    setattr(mock_settings_obj, 'MAX_EVENT_RES_DATA', 700000)
     return mock_settings_obj, mock_settings_json
 
 
diff --git a/awx/main/tests/unit/api/test_logger.py b/awx/main/tests/unit/api/test_logger.py
index d08362259b..ac1a63acbb 100644
--- a/awx/main/tests/unit/api/test_logger.py
+++ b/awx/main/tests/unit/api/test_logger.py
@@ -36,44 +36,105 @@ data_loggly = {
 # name this whatever you want
 @pytest.mark.parametrize(
     'enabled, type, host, port, protocol, expected_config', [
-        (True, 'loggly', 'http://logs-01.loggly.com/inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/', None, 'https', 
-            '''$IncludeConfig /etc/rsyslog.conf\ninput(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
-template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")
-action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" ''' +
-            'errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")'),
-        (True, 'other', 'localhost', 9000, 'udp', 
-            '''$IncludeConfig /etc/rsyslog.conf
-input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
-template(name="awx" type="string" string="%msg%")
-action(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" template="awx")'''),
-        (True, 'other', 'localhost', 9000, 'tcp', 
-            '''$IncludeConfig /etc/rsyslog.conf
-input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
-template(name="awx" type="string" string="%msg%")
-action(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" template="awx")'''),
-        (False, 'loggly', 'http://logs-01.loggly.com/inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/', 8080, 'https', 
-            '''$IncludeConfig /etc/rsyslog.conf'''),
-        (True, 'splunk', 'https://yoursplunk:8088/services/collector/event', None, None,
-            '''$IncludeConfig /etc/rsyslog.conf
-input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
-template(name="awx" type="string" string="%msg%")
-module(load="omhttp")
-action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" ''' +
-            'errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'),
-        (True, 'splunk', 'https://yoursplunk/services/collector/event', 8088, None,
-            '''$IncludeConfig /etc/rsyslog.conf
-input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
-template(name="awx" type="string" string="%msg%")
-module(load="omhttp")
-action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" ''' +
-            'errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'),
-        (True, 'splunk', 'https://yoursplunk/services/collector/event', 8088, 'https',
-            '''$IncludeConfig /etc/rsyslog.conf
-input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
-template(name="awx" type="string" string="%msg%")
-module(load="omhttp")
-action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" ''' +
-            'errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")'),
+        (
+            True,
+            'loggly',
+            'http://logs-01.loggly.com/inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/',
+            None,
+            'https',
+            '\n'.join([
+                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")',  # noqa
+            ])
+        ),
+        (
+            True,  # localhost w/ custom UDP port
+            'other',
+            'localhost',
+            9000,
+            'udp',
+            '\n'.join([
+                'template(name="awx" type="string" string="%msg%")',
+                'action(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" template="awx")',  # noqa
+            ])
+        ),
+        (
+            True,  # localhost w/ custom TCP port
+            'other',
+            'localhost',
+            9000,
+            'tcp',
+            '\n'.join([
+                'template(name="awx" type="string" string="%msg%")',
+                'action(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" template="awx")',  # noqa
+            ])
+        ),
+        (
+            True,  # https, default port 443
+            'splunk',
+            'https://yoursplunk/services/collector/event',
+            None,
+            None,
+            '\n'.join([
+                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+            ])
+        ),
+        (
+            True,  # http, default port 80
+            'splunk',
+            'http://yoursplunk/services/collector/event',
+            None,
+            None,
+            '\n'.join([
+                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+            ])
+        ),
+        (
+            True,  # https, custom port in URL string
+            'splunk',
+            'https://yoursplunk:8088/services/collector/event',
+            None,
+            None,
+            '\n'.join([
+                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+            ])
+        ),
+        (
+            True,  # https, custom port explicitly specified
+            'splunk',
+            'https://yoursplunk/services/collector/event',
+            8088,
+            None,
+            '\n'.join([
+                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+            ])
+        ),
+        (
+            True,  # no scheme specified in URL, default to https, respect custom port
+            'splunk',
+            'yoursplunk.org/services/collector/event',
+            8088,
+            'https',
+            '\n'.join([
+                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+            ])
+        ),
+        (
+            True,  # respect custom http-only port
+            'splunk',
+            'http://yoursplunk.org/services/collector/event',
+            8088,
+            None,
+            '\n'.join([
+                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+            ])
+        ),
     ]
 )
 def test_rsyslog_conf_template(enabled, type, host, port, protocol, expected_config):
@@ -96,4 +157,4 @@ def test_rsyslog_conf_template(enabled, type, host, port, protocol, expected_con
     tmpl = construct_rsyslog_conf_template(mock_settings)
     
     # check validity of created template
-    assert tmpl in expected_config
+    assert expected_config in tmpl
diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index e5e11d5156..7865a5a705 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -67,7 +67,6 @@ def construct_rsyslog_conf_template(settings=settings):
         parts.append(
             f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
         )
-    # parts.append('$IncludeConfig /var/lib/awx/rsyslog/*.conf')
     tmpl = '\n'.join(parts)
     return tmpl
 

From 5d54877183bb5dd2eb58d55e0409ea9d7e612137 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Wed, 8 Apr 2020 16:25:01 -0400
Subject: [PATCH 23/37] Add action to default rsyslog.conf so supervisor starts
 correctly the first time

---
 installer/roles/image_build/files/rsyslog.conf      | 1 +
 installer/roles/image_build/tasks/main.yml          | 7 +++++++
 installer/roles/image_build/templates/Dockerfile.j2 | 5 ++---
 installer/roles/kubernetes/templates/rsyslog.yml.j2 | 7 ++++++-
 tools/docker-compose/rsyslog.conf                   | 1 +
 5 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/installer/roles/image_build/files/rsyslog.conf b/installer/roles/image_build/files/rsyslog.conf
index c3a2d6727d..dec1f8576e 100644
--- a/installer/roles/image_build/files/rsyslog.conf
+++ b/installer/roles/image_build/files/rsyslog.conf
@@ -4,3 +4,4 @@ $IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf
 $ModLoad imuxsock
 input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
 template(name="awx" type="string" string="%msg%")
+action(type="omfile" file="/dev/null")
diff --git a/installer/roles/image_build/tasks/main.yml b/installer/roles/image_build/tasks/main.yml
index 51e9b239ab..be2a66f11d 100644
--- a/installer/roles/image_build/tasks/main.yml
+++ b/installer/roles/image_build/tasks/main.yml
@@ -144,6 +144,13 @@
     mode: '0700'
   delegate_to: localhost
 
+- name: Stage rsyslog.conf
+  copy:
+    src: rsyslog.conf
+    dest: "{{ docker_base_path }}/rsyslog.conf"
+    mode: '0700'
+  delegate_to: localhost
+
 - name: Stage supervisor.conf
   copy:
     src: supervisor.conf
diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2
index 50ec4686e1..487cb4edbe 100644
--- a/installer/roles/image_build/templates/Dockerfile.j2
+++ b/installer/roles/image_build/templates/Dockerfile.j2
@@ -97,7 +97,7 @@ RUN cd /usr/local/bin && \
     curl -L https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz | \
     tar -xz --strip-components=1 --wildcards --no-anchored 'oc'
 
-ADD rsyslog.repo /etc/yum.repos.d/
+ADD rsyslog.repo /etc/yum.repos.d/rsyslog.repo
 RUN yum install -y rsyslog-omhttp
 
 # Pre-create things that we need to write to
@@ -115,10 +115,9 @@ RUN find /var/lib/awx -not -path '/var/lib/awx/venv*' | xargs chgrp root && \
     chmod +rx /usr/bin/config-watcher && \
     chmod u+s /usr/bin/bwrap # https://github.com/ansible/awx/issues/5224
 
-# Create default awx rsyslog.conf
+# Create default awx rsyslog config
 ADD rsyslog.conf /var/lib/awx/rsyslog/rsyslog.conf
 
-
 RUN ln -sf /dev/stdout /var/log/nginx/access.log && \
     ln -sf /dev/stderr /var/log/nginx/error.log
 
diff --git a/installer/roles/kubernetes/templates/rsyslog.yml.j2 b/installer/roles/kubernetes/templates/rsyslog.yml.j2
index 4a3bbea968..3a047b4c1f 100644
--- a/installer/roles/kubernetes/templates/rsyslog.yml.j2
+++ b/installer/roles/kubernetes/templates/rsyslog.yml.j2
@@ -6,4 +6,9 @@ metadata:
 data:
   rsyslog-config: |
     $WorkDirectory /var/lib/awx/rsyslog
-    
+    $MaxMessageSize 700000
+    $IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf
+    $ModLoad imuxsock
+    input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
+    template(name="awx" type="string" string="%msg%")
+    action(type="omfile" file="/dev/null")
diff --git a/tools/docker-compose/rsyslog.conf b/tools/docker-compose/rsyslog.conf
index c3a2d6727d..dec1f8576e 100644
--- a/tools/docker-compose/rsyslog.conf
+++ b/tools/docker-compose/rsyslog.conf
@@ -4,3 +4,4 @@ $IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf
 $ModLoad imuxsock
 input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
 template(name="awx" type="string" string="%msg%")
+action(type="omfile" file="/dev/null")

From f70a76109c29ba1155fbec462b62f095449f89d8 Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Wed, 8 Apr 2020 17:55:51 -0400
Subject: [PATCH 24/37] make rsyslog fall back to no-op if logging is disabled

---
 awx/main/utils/external_logging.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 7865a5a705..ec8b279048 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -25,7 +25,6 @@ def construct_rsyslog_conf_template(settings=settings):
                 port = parsed.port
         except ValueError:
             port = settings.LOG_AGGREGATOR_PORT
-    
     max_bytes = settings.MAX_EVENT_RES_DATA
     parts.extend([
         '$WorkDirectory /var/lib/awx/rsyslog',
@@ -67,6 +66,8 @@ def construct_rsyslog_conf_template(settings=settings):
         parts.append(
             f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" template="awx")'  # noqa
         )
+    else:
+        parts.append(f'action(type="omfile" file="/dev/null")')  # rsyslog needs *at least* one valid action to start
     tmpl = '\n'.join(parts)
     return tmpl
 

From bba680671bdb261cf58afec65e228038a7c1416e Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Wed, 8 Apr 2020 20:03:33 -0400
Subject: [PATCH 25/37] when writing the rsyslog config, do it post-commit

there's a race condition if we do this pre-commit where the correct
value isn't actually *persisted* to the database yet, and we end up
saving the *prior* setting values
---
 awx/main/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index c69a0c7f78..95118c5751 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -288,7 +288,7 @@ def handle_setting_changes(setting_keys):
         setting.startswith('LOG_AGGREGATOR')
         for setting in setting_keys
     ]):
-        reconfigure_rsyslog()
+        connection.on_commit(reconfigure_rsyslog)
 
 
 @task(queue='tower_broadcast_all')

From 269558876ea5dd75918348f60c955e0a31d7a557 Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Wed, 8 Apr 2020 20:11:07 -0400
Subject: [PATCH 26/37] only use a basic auth password for external logging if
 username is set

---
 awx/main/utils/external_logging.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index ec8b279048..32f90af8d8 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -58,7 +58,8 @@ def construct_rsyslog_conf_template(settings=settings):
         password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '')
         if username:
             params.append(f'uid="{username}"')
-        if password:
+        if username and password:
+            # you can only have a basic auth password if there's a username
             params.append(f'pwd="{password}"')
         params = ' '.join(params)
         parts.extend(['module(load="omhttp")', f'action({params})'])

From 8cdd42307cf520ee6a29a3bd3c16e03f77f3bd1a Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Wed, 8 Apr 2020 20:11:30 -0400
Subject: [PATCH 27/37] clarify that logging username/password is only valid
 for HTTP/s

---
 awx/main/conf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/awx/main/conf.py b/awx/main/conf.py
index 8fb8edceda..ce6b33217c 100644
--- a/awx/main/conf.py
+++ b/awx/main/conf.py
@@ -667,7 +667,7 @@ register(
     allow_blank=True,
     default='',
     label=_('Logging Aggregator Username'),
-    help_text=_('Username for external log aggregator (if required).'),
+    help_text=_('Username for external log aggregator (if required; HTTP/s only).'),
     category=_('Logging'),
     category_slug='logging',
     required=False,
@@ -679,7 +679,7 @@ register(
     default='',
     encrypted=True,
     label=_('Logging Aggregator Password/Token'),
-    help_text=_('Password or authentication token for external log aggregator (if required).'),
+    help_text=_('Password or authentication token for external log aggregator (if required; HTTP/s only).'),
     category=_('Logging'),
     category_slug='logging',
     required=False,

From 2a4b009f04ae7d29e111ce27de21201b37076ae4 Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Thu, 9 Apr 2020 00:42:43 -0400
Subject: [PATCH 28/37] rsyslogd: use %rawmsg-after-pri% instead of %msg%

after some prolonged RFC reading and tinkering w/ rsyslogd...

cpython's SysLogHandler doesn't emit RFC3164 formatted messages
in the format you'd expect; it's missing the ISO date, hostname, etc...
along with other header values; the handler implementation relies on you
to specify a syslog-like formatter (we've replaced all of this with our
own *custom* logstash-esque formatter that effectively outputs valid JSON
- without dates and other syslog header values prepended)

because of this unanticipated format, rsyslogd chokes when trying to
parse the message's parts;  AWX is emitting:

<priority>RAWJSON

...so the usage of `%msg%` isn't going to work for us, because rsyslog
tries to parse *all* of the possible headers (and yells, because it
can't find a date to parse):

see: https://www.rsyslog.com/files/temp/doc-indent/configuration/properties.html#message-properties

this is fine, because we don't *need* any of that message parsing
anyways; in the end, we're *just* interested in forwarding the raw
JSON/text content to the third party log handler
---
 awx/main/tests/unit/api/test_logger.py | 18 +++++++++---------
 awx/main/utils/external_logging.py     |  2 +-
 awx/main/utils/formatters.py           |  2 +-
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/awx/main/tests/unit/api/test_logger.py b/awx/main/tests/unit/api/test_logger.py
index ac1a63acbb..17408a9421 100644
--- a/awx/main/tests/unit/api/test_logger.py
+++ b/awx/main/tests/unit/api/test_logger.py
@@ -43,7 +43,7 @@ data_loggly = {
             None,
             'https',
             '\n'.join([
-                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
                 'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")',  # noqa
             ])
         ),
@@ -54,7 +54,7 @@ data_loggly = {
             9000,
             'udp',
             '\n'.join([
-                'template(name="awx" type="string" string="%msg%")',
+                'template(name="awx" type="string" string="%rawmsg-after-pri%")',
                 'action(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" template="awx")',  # noqa
             ])
         ),
@@ -65,7 +65,7 @@ data_loggly = {
             9000,
             'tcp',
             '\n'.join([
-                'template(name="awx" type="string" string="%msg%")',
+                'template(name="awx" type="string" string="%rawmsg-after-pri%")',
                 'action(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" template="awx")',  # noqa
             ])
         ),
@@ -76,7 +76,7 @@ data_loggly = {
             None,
             None,
             '\n'.join([
-                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
                 'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
             ])
         ),
@@ -87,7 +87,7 @@ data_loggly = {
             None,
             None,
             '\n'.join([
-                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
                 'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
             ])
         ),
@@ -98,7 +98,7 @@ data_loggly = {
             None,
             None,
             '\n'.join([
-                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
                 'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
             ])
         ),
@@ -109,7 +109,7 @@ data_loggly = {
             8088,
             None,
             '\n'.join([
-                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
                 'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
             ])
         ),
@@ -120,7 +120,7 @@ data_loggly = {
             8088,
             'https',
             '\n'.join([
-                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
                 'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
             ])
         ),
@@ -131,7 +131,7 @@ data_loggly = {
             8088,
             None,
             '\n'.join([
-                'template(name="awx" type="string" string="%msg%")\nmodule(load="omhttp")',
+                'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
                 'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
             ])
         ),
diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 32f90af8d8..11ea0cf53f 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -32,7 +32,7 @@ def construct_rsyslog_conf_template(settings=settings):
         '$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf',
         '$ModLoad imuxsock',
         'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on")',
-        'template(name="awx" type="string" string="%msg%")',
+        'template(name="awx" type="string" string="%rawmsg-after-pri%")',
     ])
     if protocol.startswith('http'):
         # https://github.com/rsyslog/rsyslog-doc/blob/master/source/configuration/modules/omhttp.rst
diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py
index fb08034a5e..8e3ddabf1b 100644
--- a/awx/main/utils/formatters.py
+++ b/awx/main/utils/formatters.py
@@ -97,7 +97,7 @@ class LogstashFormatterBase(logging.Formatter):
 
     @classmethod
     def serialize(cls, message):
-        return ' ' + json.dumps(message, cls=DjangoJSONEncoder) + '\000'
+        return json.dumps(message, cls=DjangoJSONEncoder) + '\n'
 
 
 class LogstashFormatter(LogstashFormatterBase):

From 1000dc10fbd92c8f965f6313ee77e0ab683fda4b Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Thu, 9 Apr 2020 13:57:44 -0400
Subject: [PATCH 29/37] an an rsyslogd config check to the logging test
 endpoint

---
 awx/conf/views.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/awx/conf/views.py b/awx/conf/views.py
index 580a0d2e52..1e4f33450c 100644
--- a/awx/conf/views.py
+++ b/awx/conf/views.py
@@ -4,6 +4,7 @@
 # Python
 import collections
 import logging
+import subprocess
 import sys
 import socket
 from socket import SHUT_RDWR
@@ -173,6 +174,14 @@ class SettingLoggingTest(GenericAPIView):
         
         hostname = getattr(settings, 'LOG_AGGREGATOR_HOST', None)
         protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', None)
+
+        try:
+            subprocess.check_output(
+                ['rsyslogd', '-N1', '-f', '/var/lib/awx/rsyslog/rsyslog.conf'],
+                stderr=subprocess.STDOUT
+            )
+        except subprocess.CalledProcessError as exc:
+            return Response({'error': exc.output}, status=status.HTTP_400_BAD_REQUEST)
         
         # Check to ensure port is open at host
         if protocol in ['udp', 'tcp']:

From b0db2b7bec074eb72aa94e2bfe52678b9f443f33 Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Thu, 9 Apr 2020 13:58:48 -0400
Subject: [PATCH 30/37] add some exception handling for dealing with logging
 connection resets

when rsyslogd restarts due to config changes, there's a brief moment
where the socket will refuse connections on teardown; exception handling
is needed here to deal with that
---
 awx/main/utils/handlers.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py
index ebf299a9f0..ae0e83a9c5 100644
--- a/awx/main/utils/handlers.py
+++ b/awx/main/utils/handlers.py
@@ -18,7 +18,14 @@ class RSysLogHandler(logging.handlers.SysLogHandler):
             return
         if not os.path.exists(settings.LOGGING['handlers']['external_logger']['address']):
             return
-        return super(RSysLogHandler, self).emit(msg)
+        try:
+            return super(RSysLogHandler, self).emit(msg)
+        except ConnectionRefusedError:
+            # rsyslogd has gone to lunch; this generally means that it's just
+            # been restarted (due to a configuration change)
+            # unfortunately, we can't log that because...rsyslogd is down (and
+            # would just us back ddown this code path)
+            pass
 
 
 ColorHandler = logging.StreamHandler

From bb5136cdae289c352bafd9fb1e8775d313cb5e79 Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Thu, 9 Apr 2020 15:21:02 -0400
Subject: [PATCH 31/37] properly escape URL paths and querystrings for paths in
 logging settings

---
 awx/main/utils/external_logging.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 11ea0cf53f..4f9d53b5f4 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -53,7 +53,10 @@ def construct_rsyslog_conf_template(settings=settings):
             'healthchecktimeout="20000"',
         ]
         if parsed.path:
-            params.append(f'restpath="{parsed.path[1:]}"')
+            path = urlparse.quote(parsed.path[1:])
+            if parsed.query:
+                path = f'{path}?{urlparse.quote(parsed.query)}'
+            params.append(f'restpath="{path}"')
         username = getattr(settings, 'LOG_AGGREGATOR_USERNAME', '')
         password = getattr(settings, 'LOG_AGGREGATOR_PASSWORD', '')
         if username:

From e52cebc28e436a6c46ddd4a1a65a8046efed998d Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Thu, 9 Apr 2020 00:42:43 -0400
Subject: [PATCH 32/37] rsyslogd: use %rawmsg-after-pri% instead of %msg%

after some prolonged RFC reading and tinkering w/ rsyslogd...

cpython's SysLogHandler doesn't emit RFC3164 formatted messages
in the format you'd expect; it's missing the ISO date, hostname, etc...
along with other header values; the handler implementation relies on you
to specify a syslog-like formatter (we've replaced all of this with our
own *custom* logstash-esque formatter that effectively outputs valid JSON
- without dates and other syslog header values prepended)

because of this unanticipated format, rsyslogd chokes when trying to
parse the message's parts;  AWX is emitting:

<priority>RAWJSON

...so the usage of `%msg%` isn't going to work for us, because rsyslog
tries to parse *all* of the possible headers (and yells, because it
can't find a date to parse):

see: https://www.rsyslog.com/files/temp/doc-indent/configuration/properties.html#message-properties

this is fine, because we don't *need* any of that message parsing
anyways; in the end, we're *just* interested in forwarding the raw
JSON/text content to the third party log handler
---
 .../kubernetes/templates/deployment.yml.j2     | 18 ------------------
 .../roles/kubernetes/templates/rsyslog.yml.j2  | 14 --------------
 2 files changed, 32 deletions(-)
 delete mode 100644 installer/roles/kubernetes/templates/rsyslog.yml.j2

diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2
index 080a2ae80a..30afe9f0d5 100644
--- a/installer/roles/kubernetes/templates/deployment.yml.j2
+++ b/installer/roles/kubernetes/templates/deployment.yml.j2
@@ -98,8 +98,6 @@ spec:
               mountPath: "/var/run/supervisor"
             - name: rsyslog-socket
               mountPath: "/var/run/rsyslog"
-            - name: rsyslog-dir
-              mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
               mountPath: "/etc/pki/ca-trust/source/anchors/"
@@ -148,11 +146,6 @@ spec:
               subPath: supervisor_task.conf
               readOnly: true
 
-            - name: {{ kubernetes_deployment_name }}-rsyslog-config
-              mountPath: "/var/lib/awx/rsyslog/rsyslog.conf"
-              subPath: rsyslog.conf
-              readOnly: true
-
             - name: {{ kubernetes_deployment_name }}-secret-key
               mountPath: "/etc/tower/SECRET_KEY"
               subPath: SECRET_KEY
@@ -189,8 +182,6 @@ spec:
               mountPath: "/var/run/supervisor"
             - name: rsyslog-socket
               mountPath: "/var/run/rsyslog"
-            - name: rsyslog-dir
-              mountPath: "/var/lib/awx/rsyslog/"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
               mountPath: "/etc/pki/ca-trust/source/anchors/"
@@ -336,8 +327,6 @@ spec:
           emptyDir: {}
         - name: rsyslog-socket
           emptyDir: {}
-        - name: rsyslog-dir
-          emptyDir: {}
 {% if ca_trust_dir is defined %}
         - name: {{ kubernetes_deployment_name }}-ca-trust-dir
           hostPath:
@@ -414,13 +403,6 @@ spec:
               - key: supervisor-task-config
                 path: 'supervisor_task.conf'
 
-        - name: {{ kubernetes_deployment_name }}-rsyslog-config
-          configMap:
-            name: {{ kubernetes_deployment_name }}-rsyslog-config
-            items:
-              - key: rsyslog-config
-                path: 'rsyslog.conf'
-
         - name: {{ kubernetes_deployment_name }}-secret-key
           secret:
             secretName: "{{ kubernetes_deployment_name }}-secrets"
diff --git a/installer/roles/kubernetes/templates/rsyslog.yml.j2 b/installer/roles/kubernetes/templates/rsyslog.yml.j2
deleted file mode 100644
index 3a047b4c1f..0000000000
--- a/installer/roles/kubernetes/templates/rsyslog.yml.j2
+++ /dev/null
@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ kubernetes_deployment_name }}-rsyslog-config
-  namespace: {{ kubernetes_namespace }}
-data:
-  rsyslog-config: |
-    $WorkDirectory /var/lib/awx/rsyslog
-    $MaxMessageSize 700000
-    $IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf
-    $ModLoad imuxsock
-    input(type="imuxsock" Socket="/var/run/rsyslog/rsyslog.sock" unlink="on")
-    template(name="awx" type="string" string="%msg%")
-    action(type="omfile" file="/dev/null")

From a0e31b9c01c974d255dd523cfc39d6382a354e2d Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Mon, 13 Apr 2020 12:19:10 -0400
Subject: [PATCH 33/37] Map logging timeout value to healthchecktimeout for
 http in rsyslog config

---
 awx/main/tests/unit/api/test_logger.py    | 14 +++++++-------
 awx/main/utils/external_logging.py        |  5 +++--
 installer/roles/kubernetes/tasks/main.yml |  2 --
 tools/docker-compose.yml                  |  1 -
 4 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/awx/main/tests/unit/api/test_logger.py b/awx/main/tests/unit/api/test_logger.py
index 17408a9421..7ff9a39b02 100644
--- a/awx/main/tests/unit/api/test_logger.py
+++ b/awx/main/tests/unit/api/test_logger.py
@@ -44,7 +44,7 @@ data_loggly = {
             'https',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")',  # noqa
+                'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" healthchecktimeout="5000" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")',  # noqa
             ])
         ),
         (
@@ -77,7 +77,7 @@ data_loggly = {
             None,
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" healthchecktimeout="5000" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -88,7 +88,7 @@ data_loggly = {
             None,
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" healthchecktimeout="5000" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -99,7 +99,7 @@ data_loggly = {
             None,
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" healthchecktimeout="5000" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -110,7 +110,7 @@ data_loggly = {
             None,
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" healthchecktimeout="5000" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -121,7 +121,7 @@ data_loggly = {
             'https',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" healthchecktimeout="5000" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -132,7 +132,7 @@ data_loggly = {
             None,
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/external.err" healthchecktimeout="20000" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" healthchecktimeout="5000" restpath="services/collector/event")',  # noqa
             ])
         ),
     ]
diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py
index 4f9d53b5f4..f3d747b300 100644
--- a/awx/main/utils/external_logging.py
+++ b/awx/main/utils/external_logging.py
@@ -11,6 +11,7 @@ def construct_rsyslog_conf_template(settings=settings):
     host = getattr(settings, 'LOG_AGGREGATOR_HOST', '')
     port = getattr(settings, 'LOG_AGGREGATOR_PORT', '')
     protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '')
+    timeout = str(getattr(settings, 'LOG_AGGREGATOR_TCP_TIMEOUT', 5) * 1000)
     if protocol.startswith('http'):
         scheme = 'https'
         # urlparse requires '//' to be provided if scheme is not specified
@@ -49,8 +50,8 @@ def construct_rsyslog_conf_template(settings=settings):
             f'skipverifyhost="{skip_verify}"',
             'action.resumeRetryCount="-1"',
             'template="awx"',
-            'errorfile="/var/log/tower/external.err"',
-            'healthchecktimeout="20000"',
+            'errorfile="/var/log/tower/rsyslog.err"',
+            f'healthchecktimeout="{timeout}"',
         ]
         if parsed.path:
             path = urlparse.quote(parsed.path[1:])
diff --git a/installer/roles/kubernetes/tasks/main.yml b/installer/roles/kubernetes/tasks/main.yml
index 84a21ffa16..3ff39968b4 100644
--- a/installer/roles/kubernetes/tasks/main.yml
+++ b/installer/roles/kubernetes/tasks/main.yml
@@ -214,7 +214,6 @@
     - 'deployment'
     - 'supervisor'
     - 'launch_awx'
-    - 'rsyslog'
   no_log: true
 
 - name: Apply Deployment
@@ -226,7 +225,6 @@
     - "{{ deployment }}"
     - "{{ supervisor }}"
     - "{{ launch_awx }}"
-    - "{{ rsyslog }}"
   no_log: true
 
 - name: Delete any existing management pod
diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml
index 4b4de2e596..6a39ed5f13 100644
--- a/tools/docker-compose.yml
+++ b/tools/docker-compose.yml
@@ -9,7 +9,6 @@ services:
     hostname: awx
     command: launch_awx.sh
     environment:
-      PYTHONUNBUFFERED: 1
       CURRENT_UID:
       OS:
       SDB_HOST: 0.0.0.0

From ca7c840d8c80d87763f860d08b0585fcfbce7943 Mon Sep 17 00:00:00 2001
From: Christian Adams <chadams@redhat.com>
Date: Mon, 13 Apr 2020 19:33:23 -0400
Subject: [PATCH 34/37] Fix permissions on rsyslog.conf for k8s

---
 awx/main/constants.py                                  | 2 +-
 installer/roles/image_build/tasks/main.yml             | 2 +-
 installer/roles/kubernetes/templates/deployment.yml.j2 | 6 ++++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/awx/main/constants.py b/awx/main/constants.py
index 4c98d264dd..c32280df08 100644
--- a/awx/main/constants.py
+++ b/awx/main/constants.py
@@ -38,7 +38,7 @@ ENV_BLACKLIST = frozenset((
     'AD_HOC_COMMAND_ID', 'REST_API_URL', 'REST_API_TOKEN', 'MAX_EVENT_RES',
     'CALLBACK_QUEUE', 'CALLBACK_CONNECTION', 'CACHE',
     'JOB_CALLBACK_DEBUG', 'INVENTORY_HOSTVARS',
-    'AWX_HOST', 'PROJECT_REVISION'
+    'AWX_HOST', 'PROJECT_REVISION', 'SUPERVISOR_WEB_CONFIG_PATH'
 ))
 
 # loggers that may be called in process of emitting a log
diff --git a/installer/roles/image_build/tasks/main.yml b/installer/roles/image_build/tasks/main.yml
index be2a66f11d..428076782d 100644
--- a/installer/roles/image_build/tasks/main.yml
+++ b/installer/roles/image_build/tasks/main.yml
@@ -148,7 +148,7 @@
   copy:
     src: rsyslog.conf
     dest: "{{ docker_base_path }}/rsyslog.conf"
-    mode: '0700'
+    mode: '0660'
   delegate_to: localhost
 
 - name: Stage supervisor.conf
diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2
index 30afe9f0d5..0b6313987d 100644
--- a/installer/roles/kubernetes/templates/deployment.yml.j2
+++ b/installer/roles/kubernetes/templates/deployment.yml.j2
@@ -98,6 +98,8 @@ spec:
               mountPath: "/var/run/supervisor"
             - name: rsyslog-socket
               mountPath: "/var/run/rsyslog"
+            - name: rsyslog-dir
+              mountPath: "/var/lib/awx/rsyslog"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
               mountPath: "/etc/pki/ca-trust/source/anchors/"
@@ -182,6 +184,8 @@ spec:
               mountPath: "/var/run/supervisor"
             - name: rsyslog-socket
               mountPath: "/var/run/rsyslog"
+            - name: rsyslog-dir
+              mountPath: "/var/lib/awx/rsyslog"
 {% if ca_trust_dir is defined %}
             - name: {{ kubernetes_deployment_name }}-ca-trust-dir
               mountPath: "/etc/pki/ca-trust/source/anchors/"
@@ -327,6 +331,8 @@ spec:
           emptyDir: {}
         - name: rsyslog-socket
           emptyDir: {}
+        - name: rsyslog-dir
+          emptyDir: {}
 {% if ca_trust_dir is defined %}
         - name: {{ kubernetes_deployment_name }}-ca-trust-dir
           hostPath:

From 9440785bdd92f7005b2be2a210919b423257d95e Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Mon, 13 Apr 2020 19:45:58 -0400
Subject: [PATCH 35/37] properly set the group on the rsyslog config

---
 installer/roles/image_build/templates/Dockerfile.j2 | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2
index 487cb4edbe..22c37dd0bf 100644
--- a/installer/roles/image_build/templates/Dockerfile.j2
+++ b/installer/roles/image_build/templates/Dockerfile.j2
@@ -107,17 +107,18 @@ RUN for dir in /home/awx /var/run/supervisor /var/lib/awx /var/lib/awx/rsyslog /
     for file in /etc/passwd /var/run/nginx.pid; \
     do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done
 
+# Create default awx rsyslog config
+ADD rsyslog.conf /var/lib/awx/rsyslog/rsyslog.conf
+
 # Fix up permissions
 RUN find /var/lib/awx -not -path '/var/lib/awx/venv*' | xargs chgrp root && \
     find /var/lib/awx -not -path '/var/lib/awx/venv*' | xargs chmod g+w && \
+    chgrp root /var/lib/awx/rsyslog/rsyslog.conf && \
     chmod +rx /usr/bin/launch_awx.sh && \
     chmod +rx /usr/bin/launch_awx_task.sh && \
     chmod +rx /usr/bin/config-watcher && \
     chmod u+s /usr/bin/bwrap # https://github.com/ansible/awx/issues/5224
 
-# Create default awx rsyslog config
-ADD rsyslog.conf /var/lib/awx/rsyslog/rsyslog.conf
-
 RUN ln -sf /dev/stdout /var/log/nginx/access.log && \
     ln -sf /dev/stderr /var/log/nginx/error.log
 

From 69cf915a20ab183e1eeaf0a7231fa4d41e7bcc23 Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Mon, 13 Apr 2020 20:25:53 -0400
Subject: [PATCH 36/37] add rsyslogd block to the k8s supervisord config file

---
 .../roles/kubernetes/templates/supervisor.yml.j2   | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/installer/roles/kubernetes/templates/supervisor.yml.j2 b/installer/roles/kubernetes/templates/supervisor.yml.j2
index 2ba5ba0e27..7ae960a817 100644
--- a/installer/roles/kubernetes/templates/supervisor.yml.j2
+++ b/installer/roles/kubernetes/templates/supervisor.yml.j2
@@ -53,8 +53,20 @@ data:
     stderr_logfile=/dev/stderr
     stderr_logfile_maxbytes=0
 
+    [program:awx-rsyslogd]
+    command = rsyslogd -n -i /var/run/rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
+    autostart = true
+    autorestart = true
+    stopwaitsecs = 1
+    stopsignal=KILL
+    stopasgroup=true
+    killasgroup=true
+    redirect_stderr=true
+    stdout_logfile=/dev/stderr
+    stdout_logfile_maxbytes=0
+
     [group:tower-processes]
-    programs=nginx,uwsgi,daphne,wsbroadcast
+    programs=nginx,uwsgi,daphne,wsbroadcast,awx-rsyslogd
     priority=5
 
     # TODO: Exit Handler

From f7f1bdf9c90d02f8f1aeb354c3a54128c357c25b Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Mon, 13 Apr 2020 20:55:05 -0400
Subject: [PATCH 37/37] properly configure supervisorctl to point at the web
 volume mount

---
 installer/roles/kubernetes/templates/supervisor.yml.j2 | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/installer/roles/kubernetes/templates/supervisor.yml.j2 b/installer/roles/kubernetes/templates/supervisor.yml.j2
index 7ae960a817..407fcf2e0b 100644
--- a/installer/roles/kubernetes/templates/supervisor.yml.j2
+++ b/installer/roles/kubernetes/templates/supervisor.yml.j2
@@ -81,10 +81,10 @@ data:
     priority=0
 
     [unix_http_server]
-    file=/tmp/supervisor.sock
+    file=/var/run/supervisor/supervisor.web.sock
 
     [supervisorctl]
-    serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
+    serverurl=unix:///var/run/supervisor/supervisor.web.sock ; use a unix:// URL  for a unix socket
 
     [rpcinterface:supervisor]
     supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
@@ -140,4 +140,3 @@ data:
 
     [rpcinterface:supervisor]
     supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-