Replace logging-related restart with dynamic handler

refactor existing handlers to be the related
  "real" handler classes, which are swapped
  out dynamically by external logger "proxy" handler class

real handler swapout only done on setting change

remove restart_local_services method
get rid of uWSGI fifo file

change TCP/UDP return type contract so that it mirrors
  the request futures object
add details to socket error messages

Makefile

@@ -296,7 +296,7 @@ uwsgi: collectstatic
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-    uwsgi -b 32768 --socket 127.0.0.1:8050 --module=awx.wsgi:application --home=/venv/awx --chdir=/awx_devel/ --vacuum --processes=5 --harakiri=120 --master --no-orphans --py-autoreload 1 --max-requests=1000 --stats /tmp/stats.socket --master-fifo=/awxfifo --lazy-apps --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" --hook-accepting1-once="exec:/bin/sh -c '[ -f /tmp/celery_pid ] && kill -1 `cat /tmp/celery_pid` || true'"
+    uwsgi -b 32768 --socket 127.0.0.1:8050 --module=awx.wsgi:application --home=/venv/awx --chdir=/awx_devel/ --vacuum --processes=5 --harakiri=120 --master --no-orphans --py-autoreload 1 --max-requests=1000 --stats /tmp/stats.socket --lazy-apps --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" --hook-accepting1-once="exec:/bin/sh -c '[ -f /tmp/celery_pid ] && kill -1 `cat /tmp/celery_pid` || true'"
 
 daphne:
 	@if [ "$(VENV_BASE)" ]; then \

awx/conf/apps.py

@@ -2,8 +2,6 @@
 from django.apps import AppConfig
 # from django.core import checks
 from django.utils.translation import ugettext_lazy as _
-from awx.main.utils.handlers import configure_external_logger
-from django.conf import settings
 
 
 class ConfConfig(AppConfig):
@@ -15,4 +13,3 @@ class ConfConfig(AppConfig):
         self.module.autodiscover()
         from .settings import SettingsWrapper
         SettingsWrapper.initialize()
-        configure_external_logger(settings)

awx/conf/tests/functional/test_api.py

@@ -338,13 +338,14 @@ def test_setting_singleton_delete_no_read_only_fields(api_request, dummy_setting
 
 @pytest.mark.django_db
 def test_setting_logging_test(api_request):
-    with mock.patch('awx.conf.views.BaseHTTPSHandler.perform_test') as mock_func:
+    with mock.patch('awx.conf.views.AWXProxyHandler.perform_test') as mock_func:
         api_request(
             'post',
             reverse('api:setting_logging_test'),
             data={'LOG_AGGREGATOR_HOST': 'http://foobar', 'LOG_AGGREGATOR_TYPE': 'logstash'}
         )
-        test_arguments = mock_func.call_args[0][0]
-        assert test_arguments.LOG_AGGREGATOR_HOST == 'http://foobar'
-        assert test_arguments.LOG_AGGREGATOR_TYPE == 'logstash'
-        assert test_arguments.LOG_AGGREGATOR_LEVEL == 'DEBUG'
+        call = mock_func.call_args_list[0]
+        args, kwargs = call
+        given_settings = kwargs['custom_settings']
+        assert given_settings.LOG_AGGREGATOR_HOST == 'http://foobar'
+        assert given_settings.LOG_AGGREGATOR_TYPE == 'logstash'

awx/conf/views.py

@@ -21,7 +21,7 @@ from awx.api.generics import *  # noqa
 from awx.api.permissions import IsSuperUser
 from awx.api.versioning import reverse, get_request_version
 from awx.main.utils import *  # noqa
-from awx.main.utils.handlers import BaseHTTPSHandler, UDPHandler, LoggingConnectivityException
+from awx.main.utils.handlers import AWXProxyHandler, LoggingConnectivityException
 from awx.main.tasks import handle_setting_changes
 from awx.conf.license import get_licensed_features
 from awx.conf.models import Setting
@@ -198,12 +198,9 @@ class SettingLoggingTest(GenericAPIView):
             mock_settings = MockSettings()
             for k, v in serializer.validated_data.items():
                 setattr(mock_settings, k, v)
-            mock_settings.LOG_AGGREGATOR_LEVEL = 'DEBUG'
+            AWXProxyHandler().perform_test(custom_settings=mock_settings)
             if mock_settings.LOG_AGGREGATOR_PROTOCOL.upper() == 'UDP':
-                UDPHandler.perform_test(mock_settings)
                 return Response(status=status.HTTP_201_CREATED)
-            else:
-                BaseHTTPSHandler.perform_test(mock_settings)
         except LoggingConnectivityException as e:
             return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
         return Response(status=status.HTTP_200_OK)

awx/main/tasks.py

@@ -29,7 +29,7 @@ except Exception:
 
 # Celery
 from celery import Task, shared_task, Celery
-from celery.signals import celeryd_init, worker_process_init, worker_shutdown, worker_ready, celeryd_after_setup
+from celery.signals import celeryd_init, worker_shutdown, worker_ready, celeryd_after_setup
 
 # Django
 from django.conf import settings
@@ -59,10 +59,9 @@ from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field,
                             wrap_args_with_proot, OutputEventFilter, OutputVerboseFilter, ignore_inventory_computed_fields,
                             ignore_inventory_group_removal, get_type_for_model, extract_ansible_vars)
 from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja
-from awx.main.utils.reload import restart_local_services, stop_local_services
+from awx.main.utils.reload import stop_local_services
 from awx.main.utils.pglock import advisory_lock
 from awx.main.utils.ha import update_celery_worker_routes, register_celery_worker_queues
-from awx.main.utils.handlers import configure_external_logger
 from awx.main.consumers import emit_channel_notification
 from awx.conf import settings_registry
 
@@ -117,15 +116,6 @@ def celery_startup(conf=None, **kwargs):
             logger.exception(six.text_type("Failed to rebuild schedule {}.").format(sch))
 
 
-@worker_process_init.connect
-def task_set_logger_pre_run(*args, **kwargs):
-    try:
-        cache.close()
-        configure_external_logger(settings, is_startup=False)
-    except Exception:
-        logger.exception('Encountered error on initial log configuration.')
-
-
 @worker_shutdown.connect
 def inform_cluster_of_shutdown(*args, **kwargs):
     try:
@@ -200,10 +190,6 @@ def handle_setting_changes(self, setting_keys):
     cache_keys = set(setting_keys)
     logger.debug('cache delete_many(%r)', cache_keys)
     cache.delete_many(cache_keys)
-    for key in cache_keys:
-        if key.startswith('LOG_AGGREGATOR_'):
-            restart_local_services(['uwsgi', 'celery', 'beat', 'callback'])
-            break
 
 
 @shared_task(bind=True, exchange='tower_broadcast_all')

awx/main/tests/functional/api/test_settings.py

@@ -14,7 +14,7 @@ import mock
 # AWX
 from awx.api.versioning import reverse
 from awx.conf.models import Setting
-from awx.main.utils.handlers import BaseHTTPSHandler, LoggingConnectivityException
+from awx.main.utils.handlers import AWXProxyHandler, LoggingConnectivityException
 
 import six
 
@@ -217,7 +217,7 @@ def test_logging_aggregrator_connection_test_bad_request(get, post, admin, key):
 
 @pytest.mark.django_db
 def test_logging_aggregrator_connection_test_valid(mocker, get, post, admin):
-    with mock.patch.object(BaseHTTPSHandler, 'perform_test') as perform_test:
+    with mock.patch.object(AWXProxyHandler, 'perform_test') as perform_test:
         url = reverse('api:setting_logging_test')
         user_data = {
             'LOG_AGGREGATOR_TYPE': 'logstash',
@@ -227,7 +227,8 @@ def test_logging_aggregrator_connection_test_valid(mocker, get, post, admin):
             'LOG_AGGREGATOR_PASSWORD': 'mcstash'
         }
         post(url, user_data, user=admin, expect=200)
-        create_settings = perform_test.call_args[0][0]
+        args, kwargs = perform_test.call_args_list[0]
+        create_settings = kwargs['custom_settings']
         for k, v in user_data.items():
             assert hasattr(create_settings, k)
             assert getattr(create_settings, k) == v
@@ -238,7 +239,7 @@ def test_logging_aggregrator_connection_test_with_masked_password(mocker, patch,
     url = reverse('api:setting_singleton_detail', kwargs={'category_slug': 'logging'})
     patch(url, user=admin, data={'LOG_AGGREGATOR_PASSWORD': 'password123'}, expect=200)
 
-    with mock.patch.object(BaseHTTPSHandler, 'perform_test') as perform_test:
+    with mock.patch.object(AWXProxyHandler, 'perform_test') as perform_test:
         url = reverse('api:setting_logging_test')
         user_data = {
             'LOG_AGGREGATOR_TYPE': 'logstash',
@@ -248,13 +249,14 @@ def test_logging_aggregrator_connection_test_with_masked_password(mocker, patch,
             'LOG_AGGREGATOR_PASSWORD': '$encrypted$'
         }
         post(url, user_data, user=admin, expect=200)
-        create_settings = perform_test.call_args[0][0]
+        args, kwargs = perform_test.call_args_list[0]
+        create_settings = kwargs['custom_settings']
         assert getattr(create_settings, 'LOG_AGGREGATOR_PASSWORD') == 'password123'
 
 
 @pytest.mark.django_db
 def test_logging_aggregrator_connection_test_invalid(mocker, get, post, admin):
-    with mock.patch.object(BaseHTTPSHandler, 'perform_test') as perform_test:
+    with mock.patch.object(AWXProxyHandler, 'perform_test') as perform_test:
         perform_test.side_effect = LoggingConnectivityException('404: Not Found')
         url = reverse('api:setting_logging_test')
         resp = post(url, {

awx/main/tests/unit/conftest.py

@@ -1,4 +1,5 @@
 import pytest
+import logging
 
 from mock import PropertyMock
 
@@ -7,3 +8,16 @@ from mock import PropertyMock
 def _disable_database_settings(mocker):
     m = mocker.patch('awx.conf.settings.SettingsWrapper.all_supported_settings', new_callable=PropertyMock)
     m.return_value = []
+
+
+@pytest.fixture()
+def dummy_log_record():
+    return logging.LogRecord(
+        'awx', # logger name
+        20, # loglevel INFO
+        './awx/some/module.py', # pathname
+        100, # lineno
+        'User joe logged in', # msg
+        tuple(), # args,
+        None # exc_info
+    )

awx/main/tests/unit/utils/test_filters.py

@@ -5,7 +5,7 @@ import mock
 from collections import namedtuple
 
 # AWX
-from awx.main.utils.filters import SmartFilter
+from awx.main.utils.filters import SmartFilter, ExternalLoggerEnabled
 
 # Django
 from django.db.models import Q
@@ -13,6 +13,37 @@ from django.db.models import Q
 import six
 
 
+@pytest.mark.parametrize('params, logger_name, expected', [
+    # skip all records if enabled_flag = False
+    ({'enabled_flag': False}, 'awx.main', False),
+    # skip all records if the host is undefined
+    ({'enabled_flag': True}, 'awx.main', False),
+    # skip all records if underlying logger is used by handlers themselves
+    ({'enabled_flag': True}, 'awx.main.utils.handlers', False),
+    ({'enabled_flag': True, 'enabled_loggers': ['awx']}, 'awx.main', True),
+    ({'enabled_flag': True, 'enabled_loggers': ['abc']}, 'awx.analytics.xyz', False),
+    ({'enabled_flag': True, 'enabled_loggers': ['xyz']}, 'awx.analytics.xyz', True),
+])
+def test_base_logging_handler_skip_log(params, logger_name, expected, dummy_log_record):
+    filter = ExternalLoggerEnabled(**params)
+    dummy_log_record.name = logger_name
+    assert filter.filter(dummy_log_record) is expected, (params, logger_name)
+
+
+@pytest.mark.parametrize('level, expect', [
+    (30, True),  # warning
+    (20, False)  # info
+])
+def test_log_configurable_severity(level, expect, dummy_log_record):
+    dummy_log_record.levelno = level
+    filter = ExternalLoggerEnabled(
+        enabled_flag=True,
+        enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'],
+        lvl='WARNING'
+    )
+    assert filter.filter(dummy_log_record) is expect
+
+
 Field = namedtuple('Field', 'name')
 Meta = namedtuple('Meta', 'fields')
 

awx/main/tests/unit/utils/test_handlers.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 import base64
 import cStringIO
-import json
 import logging
 import socket
 import datetime
@@ -10,7 +9,6 @@ from uuid import uuid4
 
 import mock
 
-from django.conf import settings
 from django.conf import LazySettings
 import pytest
 import requests
@@ -18,23 +16,11 @@ from requests_futures.sessions import FuturesSession
 
 from awx.main.utils.handlers import (BaseHandler, BaseHTTPSHandler as HTTPSHandler,
                                      TCPHandler, UDPHandler, _encode_payload_for_socket,
-                                     PARAM_NAMES, LoggingConnectivityException)
+                                     PARAM_NAMES, LoggingConnectivityException,
+                                     AWXProxyHandler)
 from awx.main.utils.formatters import LogstashFormatter
 
 
-@pytest.fixture()
-def dummy_log_record():
-    return logging.LogRecord(
-        'awx', # logger name
-        20, # loglevel INFO
-        './awx/some/module.py', # pathname
-        100, # lineno
-        'User joe logged in', # msg
-        tuple(), # args,
-        None # exc_info
-    )
-
-
 @pytest.fixture()
 def http_adapter():
     class FakeHTTPAdapter(requests.adapters.HTTPAdapter):
@@ -80,105 +66,91 @@ def test_https_logging_handler_requests_async_implementation():
 
 
 def test_https_logging_handler_has_default_http_timeout():
-    handler = HTTPSHandler.from_django_settings(settings)
+    handler = TCPHandler()
     assert handler.tcp_timeout == 5
 
 
-@pytest.mark.parametrize('param', PARAM_NAMES.keys())
+@pytest.mark.parametrize('param', ['host', 'port', 'indv_facts'])
 def test_base_logging_handler_defaults(param):
     handler = BaseHandler()
     assert hasattr(handler, param) and getattr(handler, param) is None
 
 
-@pytest.mark.parametrize('param', PARAM_NAMES.keys())
+@pytest.mark.parametrize('param', ['host', 'port', 'indv_facts'])
 def test_base_logging_handler_kwargs(param):
     handler = BaseHandler(**{param: 'EXAMPLE'})
     assert hasattr(handler, param) and getattr(handler, param) == 'EXAMPLE'
 
 
-@pytest.mark.parametrize('param, django_settings_name', PARAM_NAMES.items())
-def test_base_logging_handler_from_django_settings(param, django_settings_name):
+@pytest.mark.parametrize('params', [
+    {
+        'LOG_AGGREGATOR_HOST': 'https://server.invalid',
+        'LOG_AGGREGATOR_PORT': 22222,
+        'LOG_AGGREGATOR_TYPE': 'loggly',
+        'LOG_AGGREGATOR_USERNAME': 'foo',
+        'LOG_AGGREGATOR_PASSWORD': 'bar',
+        'LOG_AGGREGATOR_INDIVIDUAL_FACTS': True,
+        'LOG_AGGREGATOR_TCP_TIMEOUT': 96,
+        'LOG_AGGREGATOR_VERIFY_CERT': False,
+        'LOG_AGGREGATOR_PROTOCOL': 'https'
+    },
+    {
+        'LOG_AGGREGATOR_HOST': 'https://server.invalid',
+        'LOG_AGGREGATOR_PORT': 22222,
+        'LOG_AGGREGATOR_PROTOCOL': 'udp'
+    }
+])
+def test_real_handler_from_django_settings(params):
+    settings = LazySettings()
+    settings.configure(**params)
+    handler = AWXProxyHandler().get_handler(custom_settings=settings)
+    # need the _reverse_ dictionary from PARAM_NAMES
+    attr_lookup = {}
+    for attr_name, setting_name in PARAM_NAMES.items():
+        attr_lookup[setting_name] = attr_name
+    for setting_name, val in params.items():
+        attr_name = attr_lookup[setting_name]
+        if attr_name == 'protocol':
+            continue
+        assert hasattr(handler, attr_name)
+
+
+def test_invalid_kwarg_to_real_handler():
     settings = LazySettings()
     settings.configure(**{
-        django_settings_name: 'EXAMPLE'
+        'LOG_AGGREGATOR_HOST': 'https://server.invalid',
+        'LOG_AGGREGATOR_PORT': 22222,
+        'LOG_AGGREGATOR_PROTOCOL': 'udp',
+        'LOG_AGGREGATOR_VERIFY_CERT': False  # setting not valid for UDP handler
     })
-    handler = BaseHandler.from_django_settings(settings)
-    assert hasattr(handler, param) and getattr(handler, param) == 'EXAMPLE'
+    handler = AWXProxyHandler().get_handler(custom_settings=settings)
+    assert not hasattr(handler, 'verify_cert')
 
 
-@pytest.mark.parametrize('params, logger_name, expected', [
-    # skip all records if enabled_flag = False
-    ({'enabled_flag': False}, 'awx.main', True),
-    # skip all records if the host is undefined
-    ({'host': '', 'enabled_flag': True}, 'awx.main', True),
-    # skip all records if underlying logger is used by handlers themselves
-    ({'host': '127.0.0.1', 'enabled_flag': True}, 'awx.main.utils.handlers', True),
-    ({'host': '127.0.0.1', 'enabled_flag': True}, 'awx.main', False),
-    ({'host': '127.0.0.1', 'enabled_flag': True, 'enabled_loggers': ['abc']}, 'awx.analytics.xyz', True),
-    ({'host': '127.0.0.1', 'enabled_flag': True, 'enabled_loggers': ['xyz']}, 'awx.analytics.xyz', False),
-])
-def test_base_logging_handler_skip_log(params, logger_name, expected):
-    handler = BaseHandler(**params)
-    assert handler._skip_log(logger_name) is expected
-
-
-def test_base_logging_handler_emit(dummy_log_record):
-    handler = BaseHandler(host='127.0.0.1', enabled_flag=True,
-                          message_type='logstash', lvl='INFO',
-                          enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+def test_base_logging_handler_emit_system_tracking(dummy_log_record):
+    handler = BaseHandler(host='127.0.0.1', indv_facts=True)
     handler.setFormatter(LogstashFormatter())
-    sent_payloads = handler.emit(dummy_log_record)
-
-    assert len(sent_payloads) == 1
-    body = json.loads(sent_payloads[0])
-
-    assert body['level'] == 'INFO'
-    assert body['logger_name'] == 'awx'
-    assert body['message'] == 'User joe logged in'
-
-
-def test_base_logging_handler_ignore_low_severity_msg(dummy_log_record):
-    handler = BaseHandler(host='127.0.0.1', enabled_flag=True,
-                          message_type='logstash', lvl='WARNING',
-                          enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
-    handler.setFormatter(LogstashFormatter())
-    sent_payloads = handler.emit(dummy_log_record)
-    assert len(sent_payloads) == 0
-
-
-def test_base_logging_handler_emit_system_tracking():
-    handler = BaseHandler(host='127.0.0.1', enabled_flag=True,
-                          message_type='logstash', indv_facts=True, lvl='INFO',
-                          enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
-    handler.setFormatter(LogstashFormatter())
-    record = logging.LogRecord(
-        'awx.analytics.system_tracking', # logger name
-        20, # loglevel INFO
-        './awx/some/module.py', # pathname
-        100, # lineno
-        None, # msg
-        tuple(), # args,
-        None # exc_info
-    )
-    record.inventory_id = 11
-    record.host_name = 'my_lucky_host'
-    record.job_id = 777
-    record.ansible_facts = {
+    dummy_log_record.name = 'awx.analytics.system_tracking'
+    dummy_log_record.msg = None
+    dummy_log_record.inventory_id = 11
+    dummy_log_record.host_name = 'my_lucky_host'
+    dummy_log_record.job_id = 777
+    dummy_log_record.ansible_facts = {
         "ansible_kernel": "4.4.66-boot2docker",
         "ansible_machine": "x86_64",
         "ansible_swapfree_mb": 4663,
     }
-    record.ansible_facts_modified = datetime.datetime.now(tzutc()).isoformat()
-    sent_payloads = handler.emit(record)
+    dummy_log_record.ansible_facts_modified = datetime.datetime.now(tzutc()).isoformat()
+    sent_payloads = handler.emit(dummy_log_record)
 
     assert len(sent_payloads) == 1
-    assert sent_payloads[0]['ansible_facts'] == record.ansible_facts
-    assert sent_payloads[0]['ansible_facts_modified'] == record.ansible_facts_modified
+    assert sent_payloads[0]['ansible_facts'] == dummy_log_record.ansible_facts
+    assert sent_payloads[0]['ansible_facts_modified'] == dummy_log_record.ansible_facts_modified
     assert sent_payloads[0]['level'] == 'INFO'
     assert sent_payloads[0]['logger_name'] == 'awx.analytics.system_tracking'
-    assert sent_payloads[0]['job_id'] == record.job_id
-    assert sent_payloads[0]['inventory_id'] == record.inventory_id
-    assert sent_payloads[0]['host_name'] == record.host_name
+    assert sent_payloads[0]['job_id'] == dummy_log_record.job_id
+    assert sent_payloads[0]['inventory_id'] == dummy_log_record.inventory_id
+    assert sent_payloads[0]['host_name'] == dummy_log_record.host_name
 
 
 @pytest.mark.parametrize('host, port, normalized, hostname_only', [
@@ -236,16 +208,18 @@ def test_https_logging_handler_connectivity_test(http_adapter, status, reason, e
         def emit(self, record):
             return super(FakeHTTPSHandler, self).emit(record)
 
-    if exc:
-        with pytest.raises(exc) as e:
-            FakeHTTPSHandler.perform_test(settings)
-        assert str(e).endswith('%s: %s' % (status, reason))
-    else:
-        assert FakeHTTPSHandler.perform_test(settings) is None
+    with mock.patch.object(AWXProxyHandler, 'get_handler_class') as mock_get_class:
+        mock_get_class.return_value = FakeHTTPSHandler
+        if exc:
+            with pytest.raises(exc) as e:
+                AWXProxyHandler().perform_test(settings)
+            assert str(e).endswith('%s: %s' % (status, reason))
+        else:
+            assert AWXProxyHandler().perform_test(settings) is None
 
 
 def test_https_logging_handler_logstash_auth_info():
-    handler = HTTPSHandler(message_type='logstash', username='bob', password='ansible', lvl='INFO')
+    handler = HTTPSHandler(message_type='logstash', username='bob', password='ansible')
     handler._add_auth_information()
     assert isinstance(handler.session.auth, requests.auth.HTTPBasicAuth)
     assert handler.session.auth.username == 'bob'
@@ -261,9 +235,7 @@ def test_https_logging_handler_splunk_auth_info():
 
 def test_https_logging_handler_connection_error(connection_error_adapter,
                                                 dummy_log_record):
-    handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
-                           message_type='logstash', lvl='INFO',
-                           enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+    handler = HTTPSHandler(host='127.0.0.1', message_type='logstash')
     handler.setFormatter(LogstashFormatter())
     handler.session.mount('http://', connection_error_adapter)
 
@@ -289,9 +261,7 @@ def test_https_logging_handler_connection_error(connection_error_adapter,
 @pytest.mark.parametrize('message_type', ['logstash', 'splunk'])
 def test_https_logging_handler_emit_without_cred(http_adapter, dummy_log_record,
                                                  message_type):
-    handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
-                           message_type=message_type, lvl='INFO',
-                           enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+    handler = HTTPSHandler(host='127.0.0.1', message_type=message_type)
     handler.setFormatter(LogstashFormatter())
     handler.session.mount('http://', http_adapter)
     async_futures = handler.emit(dummy_log_record)
@@ -312,10 +282,9 @@ def test_https_logging_handler_emit_without_cred(http_adapter, dummy_log_record,
 
 def test_https_logging_handler_emit_logstash_with_creds(http_adapter,
                                                         dummy_log_record):
-    handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
+    handler = HTTPSHandler(host='127.0.0.1',
                            username='user', password='pass',
-                           message_type='logstash', lvl='INFO',
-                           enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+                           message_type='logstash')
     handler.setFormatter(LogstashFormatter())
     handler.session.mount('http://', http_adapter)
     async_futures = handler.emit(dummy_log_record)
@@ -328,9 +297,8 @@ def test_https_logging_handler_emit_logstash_with_creds(http_adapter,
 
 def test_https_logging_handler_emit_splunk_with_creds(http_adapter,
                                                       dummy_log_record):
-    handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
-                           password='pass', message_type='splunk', lvl='INFO',
-                           enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+    handler = HTTPSHandler(host='127.0.0.1',
+                           password='pass', message_type='splunk')
     handler.setFormatter(LogstashFormatter())
     handler.session.mount('http://', http_adapter)
     async_futures = handler.emit(dummy_log_record)
@@ -351,9 +319,7 @@ def test_encode_payload_for_socket(payload, encoded_payload):
 
 
 def test_udp_handler_create_socket_at_init():
-    handler = UDPHandler(host='127.0.0.1', port=4399,
-                         enabled_flag=True, message_type='splunk', lvl='INFO',
-                         enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+    handler = UDPHandler(host='127.0.0.1', port=4399)
     assert hasattr(handler, 'socket')
     assert isinstance(handler.socket, socket.socket)
     assert handler.socket.family == socket.AF_INET
@@ -361,9 +327,7 @@ def test_udp_handler_create_socket_at_init():
 
 
 def test_udp_handler_send(dummy_log_record):
-    handler = UDPHandler(host='127.0.0.1', port=4399,
-                         enabled_flag=True, message_type='splunk', lvl='INFO',
-                         enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+    handler = UDPHandler(host='127.0.0.1', port=4399)
     handler.setFormatter(LogstashFormatter())
     with mock.patch('awx.main.utils.handlers._encode_payload_for_socket', return_value="des") as encode_mock,\
             mock.patch.object(handler, 'socket') as socket_mock:
@@ -373,9 +337,7 @@ def test_udp_handler_send(dummy_log_record):
 
 
 def test_tcp_handler_send(fake_socket, dummy_log_record):
-    handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5,
-                         enabled_flag=True, message_type='splunk', lvl='INFO',
-                         enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+    handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5)
     handler.setFormatter(LogstashFormatter())
     with mock.patch('socket.socket', return_value=fake_socket) as sok_init_mock,\
             mock.patch('select.select', return_value=([], [fake_socket], [])):
@@ -388,9 +350,7 @@ def test_tcp_handler_send(fake_socket, dummy_log_record):
 
 
 def test_tcp_handler_return_if_socket_unavailable(fake_socket, dummy_log_record):
-    handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5,
-                         enabled_flag=True, message_type='splunk', lvl='INFO',
-                         enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+    handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5)
     handler.setFormatter(LogstashFormatter())
     with mock.patch('socket.socket', return_value=fake_socket) as sok_init_mock,\
             mock.patch('select.select', return_value=([], [], [])):
@@ -403,9 +363,7 @@ def test_tcp_handler_return_if_socket_unavailable(fake_socket, dummy_log_record)
 
 
 def test_tcp_handler_log_exception(fake_socket, dummy_log_record):
-    handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5,
-                         enabled_flag=True, message_type='splunk', lvl='INFO',
-                         enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
+    handler = TCPHandler(host='127.0.0.1', port=4399, tcp_timeout=5)
     handler.setFormatter(LogstashFormatter())
     with mock.patch('socket.socket', return_value=fake_socket) as sok_init_mock,\
             mock.patch('select.select', return_value=([], [], [])),\

awx/main/tests/unit/utils/test_reload.py

@@ -13,31 +13,3 @@ def test_produce_supervisor_command(mocker):
             ['supervisorctl', 'restart', 'tower-processes:receiver',],
             stderr=-1, stdin=-1, stdout=-1)
 
-
-def test_routing_of_service_restarts_works(mocker):
-    '''
-    This tests that the parent restart method will call the appropriate
-    service restart methods, depending on which services are given in args
-    '''
-    with mocker.patch.object(reload, '_uwsgi_fifo_command'),\
-            mocker.patch.object(reload, '_reset_celery_thread_pool'),\
-            mocker.patch.object(reload, '_supervisor_service_command'):
-        reload.restart_local_services(['uwsgi', 'celery', 'flower', 'daphne'])
-        reload._uwsgi_fifo_command.assert_called_once_with(uwsgi_command="c")
-        reload._reset_celery_thread_pool.assert_called_once_with()
-        reload._supervisor_service_command.assert_called_once_with(['flower', 'daphne'], command="restart")
-
-
-
-def test_routing_of_service_restarts_diables(mocker):
-    '''
-    Test that methods are not called if not in the args
-    '''
-    with mocker.patch.object(reload, '_uwsgi_fifo_command'),\
-            mocker.patch.object(reload, '_reset_celery_thread_pool'),\
-            mocker.patch.object(reload, '_supervisor_service_command'):
-        reload.restart_local_services(['flower'])
-        reload._uwsgi_fifo_command.assert_not_called()
-        reload._reset_celery_thread_pool.assert_not_called()
-        reload._supervisor_service_command.assert_called_once_with(['flower'], command="restart")
-

awx/main/utils/filters.py

@@ -8,14 +8,106 @@ from pyparsing import (
     CharsNotIn,
     ParseException,
 )
+from logging import Filter, _levelNames
 
 import six
 
-import django
+from django.apps import apps
+from django.db import models
+from django.conf import settings
 
 from awx.main.utils.common import get_search_fields
 
-__all__ = ['SmartFilter']
+__all__ = ['SmartFilter', 'ExternalLoggerEnabled']
+
+
+class FieldFromSettings(object):
+    """
+    Field interface - defaults to getting value from setting
+    if otherwise set, provided value will take precedence
+        over value in settings
+    """
+
+    def __init__(self, setting_name):
+        self.setting_name = setting_name
+
+    def __get__(self, instance, type=None):
+        if self.setting_name in getattr(instance, 'settings_override', {}):
+            return instance.settings_override[self.setting_name]
+        return getattr(settings, self.setting_name, None)
+
+    def __set__(self, instance, value):
+        if value is None:
+            if hasattr(instance, 'settings_override'):
+                instance.settings_override.pop('instance', None)
+        else:
+            if not hasattr(instance, 'settings_override'):
+                instance.settings_override = {}
+            instance.settings_override[self.setting_name] = value
+
+
+class ExternalLoggerEnabled(Filter):
+
+    # Prevents recursive logging loops from swamping the server
+    LOGGER_BLACKLIST = (
+        # loggers that may be called in process of emitting a log
+        'awx.main.utils.handlers',
+        'awx.main.utils.formatters',
+        'awx.main.utils.filters',
+        'awx.main.utils.encryption',
+        'awx.main.utils.log',
+        # loggers that may be called getting logging settings
+        'awx.conf'
+    )
+
+    lvl = FieldFromSettings('LOG_AGGREGATOR_LEVEL')
+    enabled_loggers = FieldFromSettings('LOG_AGGREGATOR_LOGGERS')
+    enabled_flag = FieldFromSettings('LOG_AGGREGATOR_ENABLED')
+
+    def __init__(self, **kwargs):
+        super(ExternalLoggerEnabled, self).__init__()
+        for field_name, field_value in kwargs.items():
+            if not isinstance(ExternalLoggerEnabled.__dict__.get(field_name, None), FieldFromSettings):
+                raise Exception('%s is not a valid kwarg' % field_name)
+            if field_value is None:
+                continue
+            setattr(self, field_name, field_value)
+
+    def filter(self, record):
+        """
+        Uses the database settings to determine if the current
+        external log configuration says that this particular record
+        should be sent to the external log aggregator
+
+        False - should not be logged
+        True - should be logged
+        """
+        # Logger exceptions
+        for logger_name in self.LOGGER_BLACKLIST:
+            if record.name.startswith(logger_name):
+                return False
+        # General enablement
+        if not self.enabled_flag:
+            return False
+
+        # Level enablement
+        if record.levelno < _levelNames[self.lvl]:
+            # logging._levelNames -> logging._nameToLevel in python 3
+            return False
+
+        # Logger type enablement
+        loggers = self.enabled_loggers
+        if not loggers:
+            return False
+        if record.name.startswith('awx.analytics'):
+            base_path, headline_name = record.name.rsplit('.', 1)
+            return bool(headline_name in loggers)
+        else:
+            if '.' in record.name:
+                base_name, trailing_path = record.name.split('.', 1)
+            else:
+                base_name = record.name
+            return bool(base_name in loggers)
 
 
 def string_to_type(t):
@@ -36,7 +128,7 @@ def string_to_type(t):
 
 
 def get_model(name):
-    return django.apps.apps.get_model('main', name)
+    return apps.get_model('main', name)
 
 
 class SmartFilter(object):
@@ -52,7 +144,7 @@ class SmartFilter(object):
             search_kwargs = self._expand_search(k, v)
             if search_kwargs:
                 kwargs.update(search_kwargs)
-                q = reduce(lambda x, y: x | y, [django.db.models.Q(**{u'%s__contains' % _k:_v}) for _k, _v in kwargs.items()])
+                q = reduce(lambda x, y: x | y, [models.Q(**{u'%s__contains' % _k:_v}) for _k, _v in kwargs.items()])
                 self.result = Host.objects.filter(q)
             else:
                 kwargs[k] = v

awx/main/utils/formatters.py

@@ -9,6 +9,8 @@ import logging
 
 import six
 
+from django.conf import settings
+
 
 class TimeFormatter(logging.Formatter):
     '''
@@ -20,15 +22,6 @@ class TimeFormatter(logging.Formatter):
 
 
 class LogstashFormatter(LogstashFormatterVersion1):
-    def __init__(self, **kwargs):
-        settings_module = kwargs.pop('settings_module', None)
-        ret = super(LogstashFormatter, self).__init__(**kwargs)
-        if settings_module:
-            self.host_id = getattr(settings_module, 'CLUSTER_HOST_ID', None)
-            if hasattr(settings_module, 'LOG_AGGREGATOR_TOWER_UUID'):
-                self.tower_uuid = settings_module.LOG_AGGREGATOR_TOWER_UUID
-            self.message_type = getattr(settings_module, 'LOG_AGGREGATOR_TYPE', 'other')
-        return ret
 
     def reformat_data_for_log(self, raw_data, kind=None):
         '''
@@ -147,6 +140,15 @@ class LogstashFormatter(LogstashFormatterVersion1):
         if record.name.startswith('awx.analytics'):
             log_kind = record.name[len('awx.analytics.'):]
             fields = self.reformat_data_for_log(fields, kind=log_kind)
+        # General AWX metadata
+        for log_name, setting_name in [
+                ('type', 'LOG_AGGREGATOR_TYPE'),
+                ('cluster_host_id', 'CLUSTER_HOST_ID'),
+                ('tower_uuid', 'LOG_AGGREGATOR_TOWER_UUID')]:
+            if hasattr(settings, setting_name):
+                fields[log_name] = getattr(settings, setting_name, None)
+            elif log_name == 'type':
+                fields[log_name] = 'other'
         return fields
 
     def format(self, record):
@@ -158,18 +160,12 @@ class LogstashFormatter(LogstashFormatterVersion1):
             '@timestamp': self.format_timestamp(record.created),
             'message': record.getMessage(),
             'host': self.host,
-            'type': self.message_type,
 
             # Extra Fields
             'level': record.levelname,
             'logger_name': record.name,
         }
 
-        if getattr(self, 'tower_uuid', None):
-            message['tower_uuid'] = self.tower_uuid
-        if getattr(self, 'host_id', None):
-            message['cluster_host_id'] = self.host_id
-
         # Add extra fields
         message.update(self.get_extra_fields(record))
 

awx/main/utils/handlers.py

@@ -13,40 +13,35 @@ import six
 from concurrent.futures import ThreadPoolExecutor
 from requests.exceptions import RequestException
 
-# loggly
-import traceback
-
+# Django
 from django.conf import settings
+
+# requests futures, a dependency used by these handlers
 from requests_futures.sessions import FuturesSession
 
 # AWX
 from awx.main.utils.formatters import LogstashFormatter
 
 
-__all__ = ['HTTPSNullHandler', 'BaseHTTPSHandler', 'TCPHandler', 'UDPHandler',
-           'configure_external_logger']
+__all__ = ['BaseHTTPSHandler', 'TCPHandler', 'UDPHandler',
+           'AWXProxyHandler']
 
 
 logger = logging.getLogger('awx.main.utils.handlers')
 
-# AWX external logging handler, generally designed to be used
-# with the accompanying LogstashHandler, derives from python-logstash library
-# Non-blocking request accomplished by FuturesSession, similar
-# to the loggly-python-handler library (not used)
 
 # Translation of parameter names to names in Django settings
+# logging settings category, only those related to handler / log emission
 PARAM_NAMES = {
     'host': 'LOG_AGGREGATOR_HOST',
     'port': 'LOG_AGGREGATOR_PORT',
     'message_type': 'LOG_AGGREGATOR_TYPE',
     'username': 'LOG_AGGREGATOR_USERNAME',
     'password': 'LOG_AGGREGATOR_PASSWORD',
-    'enabled_loggers': 'LOG_AGGREGATOR_LOGGERS',
     'indv_facts': 'LOG_AGGREGATOR_INDIVIDUAL_FACTS',
-    'enabled_flag': 'LOG_AGGREGATOR_ENABLED',
     'tcp_timeout': 'LOG_AGGREGATOR_TCP_TIMEOUT',
     'verify_cert': 'LOG_AGGREGATOR_VERIFY_CERT',
-    'lvl': 'LOG_AGGREGATOR_LEVEL',
+    'protocol': 'LOG_AGGREGATOR_PROTOCOL'
 }
 
 
@@ -58,13 +53,6 @@ class LoggingConnectivityException(Exception):
     pass
 
 
-class HTTPSNullHandler(logging.NullHandler):
-    "Placeholder null handler to allow loading without database access"
-
-    def __init__(self, *args, **kwargs):
-        return super(HTTPSNullHandler, self).__init__()
-
-
 class VerboseThreadPoolExecutor(ThreadPoolExecutor):
 
     last_log_emit = 0
@@ -91,32 +79,25 @@ class VerboseThreadPoolExecutor(ThreadPoolExecutor):
                                                              **kwargs)
 
 
-LEVEL_MAPPING = {
-    'DEBUG': logging.DEBUG,
-    'INFO': logging.INFO,
-    'WARNING': logging.WARNING,
-    'ERROR': logging.ERROR,
-    'CRITICAL': logging.CRITICAL,
-}
+class SocketResult:
+    '''
+    A class to be the return type of methods that send data over a socket
+    allows object to be used in the same way as a request futures object
+    '''
+    def __init__(self, ok, reason=None):
+        self.ok = ok
+        self.reason = reason
+
+    def result(self):
+        return self
 
 
 class BaseHandler(logging.Handler):
-    def __init__(self, **kwargs):
+    def __init__(self, host=None, port=None, indv_facts=None, **kwargs):
         super(BaseHandler, self).__init__()
-        for fd in PARAM_NAMES:
-            setattr(self, fd, kwargs.get(fd, None))
-
-    @classmethod
-    def from_django_settings(cls, settings, *args, **kwargs):
-        for param, django_setting_name in PARAM_NAMES.items():
-            kwargs[param] = getattr(settings, django_setting_name, None)
-        return cls(*args, **kwargs)
-
-    def get_full_message(self, record):
-        if record.exc_info:
-            return '\n'.join(traceback.format_exception(*record.exc_info))
-        else:
-            return record.getMessage()
+        self.host = host
+        self.port = port
+        self.indv_facts = indv_facts
 
     def _send(self, payload):
         """Actually send message to log aggregator.
@@ -128,26 +109,11 @@ class BaseHandler(logging.Handler):
             return [self._send(json.loads(self.format(record)))]
         return [self._send(self.format(record))]
 
-    def _skip_log(self, logger_name):
-        if self.host == '' or (not self.enabled_flag):
-            return True
-        # Don't send handler-related records.
-        if logger_name == logger.name:
-            return True
-        # AWX log emission is only turned off by enablement setting
-        if not logger_name.startswith('awx.analytics'):
-            return False
-        return self.enabled_loggers is None or logger_name[len('awx.analytics.'):] not in self.enabled_loggers
-
     def emit(self, record):
         """
             Emit a log record.  Returns a list of zero or more
             implementation-specific objects for tests.
         """
-        if not record.name.startswith('awx.analytics') and record.levelno < LEVEL_MAPPING[self.lvl]:
-            return []
-        if self._skip_log(record.name):
-            return []
         try:
             return self._format_and_send_record(record)
         except (KeyboardInterrupt, SystemExit):
@@ -181,6 +147,11 @@ class BaseHandler(logging.Handler):
 
 
 class BaseHTTPSHandler(BaseHandler):
+    '''
+    Originally derived from python-logstash library
+    Non-blocking request accomplished by FuturesSession, similar
+    to the loggly-python-handler library
+    '''
     def _add_auth_information(self):
         if self.message_type == 'logstash':
             if not self.username:
@@ -196,39 +167,20 @@ class BaseHTTPSHandler(BaseHandler):
             }
             self.session.headers.update(headers)
 
-    def __init__(self, fqdn=False, **kwargs):
+    def __init__(self, fqdn=False, message_type=None, username=None, password=None,
+                 tcp_timeout=5, verify_cert=True, **kwargs):
         self.fqdn = fqdn
+        self.message_type = message_type
+        self.username = username
+        self.password = password
+        self.tcp_timeout = tcp_timeout
+        self.verify_cert = verify_cert
         super(BaseHTTPSHandler, self).__init__(**kwargs)
         self.session = FuturesSession(executor=VerboseThreadPoolExecutor(
             max_workers=2  # this is the default used by requests_futures
         ))
         self._add_auth_information()
 
-    @classmethod
-    def perform_test(cls, settings):
-        """
-        Tests logging connectivity for the current logging settings.
-        @raises LoggingConnectivityException
-        """
-        handler = cls.from_django_settings(settings)
-        handler.enabled_flag = True
-        handler.setFormatter(LogstashFormatter(settings_module=settings))
-        logger = logging.getLogger(__file__)
-        fn, lno, func = logger.findCaller()
-        record = logger.makeRecord('awx', 10, fn, lno,
-                                   'AWX Connection Test', tuple(),
-                                   None, func)
-        futures = handler.emit(record)
-        for future in futures:
-            try:
-                resp = future.result()
-                if not resp.ok:
-                    raise LoggingConnectivityException(
-                        ': '.join([str(resp.status_code), resp.reason or ''])
-                    )
-            except RequestException as e:
-                raise LoggingConnectivityException(str(e))
-
     def _get_post_kwargs(self, payload_input):
         if self.message_type == 'splunk':
             # Splunk needs data nested under key "event"
@@ -265,6 +217,10 @@ def _encode_payload_for_socket(payload):
 
 
 class TCPHandler(BaseHandler):
+    def __init__(self, tcp_timeout=5, **kwargs):
+        self.tcp_timeout = tcp_timeout
+        super(TCPHandler, self).__init__(**kwargs)
+
     def _send(self, payload):
         payload = _encode_payload_for_socket(payload)
         sok = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -273,39 +229,32 @@ class TCPHandler(BaseHandler):
             sok.setblocking(0)
             _, ready_to_send, _ = select.select([], [sok], [], float(self.tcp_timeout))
             if len(ready_to_send) == 0:
-                logger.warning("Socket currently busy, failed to send message")
-                sok.close()
-                return
-            sok.send(payload)
+                ret = SocketResult(False, "Socket currently busy, failed to send message")
+                logger.warning(ret.reason)
+            else:
+                sok.send(payload)
+                ret = SocketResult(True)  # success!
         except Exception as e:
-            logger.exception("Error sending message from %s: %s" %
-                             (TCPHandler.__name__, e.message))
-        sok.close()
+            ret = SocketResult(False, "Error sending message from %s: %s" %
+                               (TCPHandler.__name__,
+                                ' '.join(six.text_type(arg) for arg in e.args)))
+            logger.exception(ret.reason)
+        finally:
+            sok.close()
+        return ret
 
 
 class UDPHandler(BaseHandler):
+    message = "Cannot determine if UDP messages are received."
+
     def __init__(self, **kwargs):
         super(UDPHandler, self).__init__(**kwargs)
         self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
 
     def _send(self, payload):
         payload = _encode_payload_for_socket(payload)
-        return self.socket.sendto(payload, (self._get_host(hostname_only=True), self.port or 0))
-
-    @classmethod
-    def perform_test(cls, settings):
-        """
-        Tests logging connectivity for the current logging settings.
-        """
-        handler = cls.from_django_settings(settings)
-        handler.enabled_flag = True
-        handler.setFormatter(LogstashFormatter(settings_module=settings))
-        logger = logging.getLogger(__file__)
-        fn, lno, func = logger.findCaller()
-        record = logger.makeRecord('awx', 10, fn, lno,
-                                   'AWX Connection Test', tuple(),
-                                   None, func)
-        handler.emit(_encode_payload_for_socket(record))
+        self.socket.sendto(payload, (self._get_host(hostname_only=True), self.port or 0))
+        return SocketResult(True, reason=self.message)
 
 
 HANDLER_MAPPING = {
@@ -315,6 +264,88 @@ HANDLER_MAPPING = {
 }
 
 
+class AWXProxyHandler(logging.Handler):
+    '''
+    Handler specific to the AWX external logging feature
+
+    Will dynamically create a handler specific to the configured
+    protocol, and will create a new one automatically on setting change
+
+    Managing parameters:
+    All parameters will get their value from settings as a default
+    if the parameter was either provided on init, or set manually,
+    this value will take precedence.
+    Parameters match same parameters in the actualized handler classes.
+    '''
+
+    def __init__(self, **kwargs):
+        # TODO: process 'level' kwarg
+        super(AWXProxyHandler, self).__init__(**kwargs)
+        self._handler = None
+        self._old_kwargs = {}
+
+    def get_handler_class(self, protocol):
+        return HANDLER_MAPPING[protocol]
+
+    def get_handler(self, custom_settings=None, force_create=False):
+        new_kwargs = {}
+        use_settings = custom_settings or settings
+        for field_name, setting_name in PARAM_NAMES.items():
+            val = getattr(use_settings, setting_name, None)
+            if val is None:
+                continue
+            new_kwargs[field_name] = val
+        if new_kwargs == self._old_kwargs and self._handler and (not force_create):
+            # avoids re-creating session objects, and other such things
+            return self._handler
+        self._old_kwargs = new_kwargs.copy()
+        # TODO: remove any kwargs no applicable to that particular handler
+        protocol = new_kwargs.pop('protocol', None)
+        HandlerClass = self.get_handler_class(protocol)
+        # cleanup old handler and make new one
+        if self._handler:
+            self._handler.close()
+        logger.debug('Creating external log handler due to startup or settings change.')
+        self._handler = HandlerClass(**new_kwargs)
+        if self.formatter:
+            # self.format(record) is called inside of emit method
+            # so not safe to assume this can be handled within self
+            self._handler.setFormatter(self.formatter)
+        return self._handler
+
+    def emit(self, record):
+        actual_handler = self.get_handler()
+        return actual_handler.emit(record)
+
+    def perform_test(self, custom_settings):
+        """
+        Tests logging connectivity for given settings module.
+        @raises LoggingConnectivityException
+        """
+        handler = self.get_handler(custom_settings=custom_settings, force_create=True)
+        handler.setFormatter(LogstashFormatter())
+        logger = logging.getLogger(__file__)
+        fn, lno, func = logger.findCaller()
+        record = logger.makeRecord('awx', 10, fn, lno,
+                                   'AWX Connection Test', tuple(),
+                                   None, func)
+        futures = handler.emit(record)
+        for future in futures:
+            try:
+                resp = future.result()
+                if not resp.ok:
+                    if isinstance(resp, SocketResult):
+                        raise LoggingConnectivityException(
+                            'Socket error: {}'.format(resp.reason or '')
+                        )
+                    else:
+                        raise LoggingConnectivityException(
+                            ': '.join([str(resp.status_code), resp.reason or ''])
+                        )
+            except RequestException as e:
+                raise LoggingConnectivityException(str(e))
+
+
 ColorHandler = logging.StreamHandler
 
 if settings.COLOR_LOGS is True:
@@ -340,41 +371,3 @@ if settings.COLOR_LOGS is True:
     except ImportError:
         # logutils is only used for colored logs in the dev environment
         pass
-
-
-def _add_or_remove_logger(address, instance):
-    specific_logger = logging.getLogger(address)
-    for i, handler in enumerate(specific_logger.handlers):
-        if isinstance(handler, (HTTPSNullHandler, BaseHTTPSHandler)):
-            specific_logger.handlers[i] = instance or HTTPSNullHandler()
-            break
-    else:
-        if instance is not None:
-            specific_logger.handlers.append(instance)
-
-
-def configure_external_logger(settings_module, is_startup=True):
-    is_enabled = settings_module.LOG_AGGREGATOR_ENABLED
-    if is_startup and (not is_enabled):
-        # Pass-through if external logging not being used
-        return
-
-    instance = None
-    if is_enabled:
-        handler_class = HANDLER_MAPPING[settings_module.LOG_AGGREGATOR_PROTOCOL]
-        instance = handler_class.from_django_settings(settings_module)
-
-        # Obtain the Formatter class from settings to maintain customizations
-        configurator = logging.config.DictConfigurator(settings_module.LOGGING)
-        formatter_config = settings_module.LOGGING['formatters']['json'].copy()
-        formatter_config['settings_module'] = settings_module
-        formatter = configurator.configure_custom(formatter_config)
-
-        instance.setFormatter(formatter)
-
-    awx_logger_instance = instance
-    if is_enabled and 'awx' not in settings_module.LOG_AGGREGATOR_LOGGERS:
-        awx_logger_instance = None
-
-    _add_or_remove_logger('awx.analytics', instance)
-    _add_or_remove_logger('awx', awx_logger_instance)

awx/main/utils/reload.py

@@ -8,29 +8,9 @@ import logging
 # Django
 from django.conf import settings
 
-# Celery
-from celery import Celery
-
 logger = logging.getLogger('awx.main.utils.reload')
 
 
-def _uwsgi_fifo_command(uwsgi_command):
-    # http://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html#available-commands
-    logger.warn('Initiating uWSGI chain reload of server')
-    TRIGGER_COMMAND = uwsgi_command
-    with open(settings.UWSGI_FIFO_LOCATION, 'w') as awxfifo:
-        awxfifo.write(TRIGGER_COMMAND)
-
-
-def _reset_celery_thread_pool():
-    # Do not use current_app because of this outstanding issue:
-    # https://github.com/celery/celery/issues/4410
-    app = Celery('awx')
-    app.config_from_object('django.conf:settings')
-    app.control.broadcast('pool_restart', arguments={'reload': True},
-                          destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False)
-
-
 def _supervisor_service_command(service_internal_names, command, communicate=True):
     '''
     Service internal name options:
@@ -68,21 +48,6 @@ def _supervisor_service_command(service_internal_names, command, communicate=Tru
         logger.info('Submitted supervisorctl {} command, not waiting for result'.format(command))
 
 
-def restart_local_services(service_internal_names):
-    logger.warn('Restarting services {} on this node in response to user action'.format(service_internal_names))
-    if 'uwsgi' in service_internal_names:
-        _uwsgi_fifo_command(uwsgi_command='c')
-        service_internal_names.remove('uwsgi')
-    restart_celery = False
-    if 'celery' in service_internal_names:
-        restart_celery = True
-        service_internal_names.remove('celery')
-    _supervisor_service_command(service_internal_names, command='restart')
-    if restart_celery:
-        # Celery restarted last because this probably includes current process
-        _reset_celery_thread_pool()
-
-
 def stop_local_services(service_internal_names, communicate=True):
     logger.warn('Stopping services {} on this node in response to user action'.format(service_internal_names))
     _supervisor_service_command(service_internal_names, command='stop', communicate=communicate)

awx/settings/defaults.py

@@ -1001,6 +1001,9 @@ LOGGING = {
         'require_debug_true_or_test': {
             '()': 'awx.main.utils.RequireDebugTrueOrTest',
         },
+        'external_log_enabled': {
+            '()': 'awx.main.utils.filters.ExternalLoggerEnabled'
+        },
     },
     'formatters': {
         'simple': {
@@ -1034,11 +1037,10 @@ LOGGING = {
             'class': 'logging.NullHandler',
             'formatter': 'simple',
         },
-        'http_receiver': {
-            'class': 'awx.main.utils.handlers.HTTPSNullHandler',
-            'level': 'DEBUG',
+        'external_logger': {
+            'class': 'awx.main.utils.handlers.AWXProxyHandler',
             'formatter': 'json',
-            'host': '',
+            'filters': ['external_log_enabled'],
         },
         'mail_admins': {
             'level': 'ERROR',
@@ -1131,7 +1133,7 @@ LOGGING = {
             'handlers': ['console'],
         },
         'awx': {
-            'handlers': ['console', 'file', 'tower_warnings'],
+            'handlers': ['console', 'file', 'tower_warnings', 'external_logger'],
             'level': 'DEBUG',
         },
         'awx.conf': {
@@ -1179,7 +1181,7 @@ LOGGING = {
             'propagate': False,
         },
         'awx.analytics': {
-            'handlers': ['http_receiver'],
+            'handlers': ['external_logger'],
             'level': 'INFO',
             'propagate': False
         },

awx/settings/development.py

@@ -149,8 +149,6 @@ SERVICE_NAME_DICT = {
     "uwsgi": "uwsgi",
     "daphne": "daphne",
     "nginx": "nginx"}
-# Used for sending commands in automatic restart
-UWSGI_FIFO_LOCATION = '/awxfifo'
 
 try:
     socket.gethostbyname('docker.for.mac.internal')

awx/settings/production.py

@@ -68,8 +68,6 @@ SERVICE_NAME_DICT = {
     "channels": "awx-channels-worker",
     "uwsgi": "awx-uwsgi",
     "daphne": "awx-daphne"}
-# Used for sending commands in automatic restart
-UWSGI_FIFO_LOCATION = '/var/lib/awx/awxfifo'
 
 # Store a snapshot of default settings at this point before loading any
 # customizable config files.